diff --git a/OpenCL/m00000_a0-optimized.cl b/OpenCL/m00000_a0-optimized.cl new file mode 100644 index 000000000..884e7b816 --- /dev/null +++ b/OpenCL/m00000_a0-optimized.cl @@ -0,0 +1,296 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00000_a0-pure.cl b/OpenCL/m00000_a0-pure.cl deleted file mode 100644 index a30464522..000000000 --- a/OpenCL/m00000_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00000_a0.cl b/OpenCL/m00000_a0.cl index 884e7b816..a30464522 100644 --- a/OpenCL/m00000_a0.cl +++ b/OpenCL/m00000_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,171 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -189,108 +91,44 @@ __kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00000_a1-optimized.cl b/OpenCL/m00000_a1-optimized.cl new file mode 100644 index 000000000..1ddfb751e --- /dev/null +++ b/OpenCL/m00000_a1-optimized.cl @@ -0,0 +1,414 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_scalar.cl" +#include "inc_simd.cl" +#include "inc_hash_md5.cl" + +__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00000_a1-pure.cl b/OpenCL/m00000_a1-pure.cl deleted file mode 100644 index 76f4407f8..000000000 --- a/OpenCL/m00000_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00000_a1.cl b/OpenCL/m00000_a1.cl index 1ddfb751e..76f4407f8 100644 --- a/OpenCL/m00000_a1.cl +++ b/OpenCL/m00000_a1.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -11,231 +11,61 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_scalar.cl" -#include "inc_simd.cl" #include "inc_hash_md5.cl" -__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + md5_ctx_t ctx0; - if (gid >= gid_max) return; + md5_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -248,167 +78,33 @@ __kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00000_a3-optimized.cl b/OpenCL/m00000_a3-optimized.cl new file mode 100644 index 000000000..c775ce0e7 --- /dev/null +++ b/OpenCL/m00000_a3-optimized.cl @@ -0,0 +1,653 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" +#include "inc_hash_md5.cl" + +#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= f (b, c, d); \ + a -= x; \ + a -= t; \ +} + +#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= x; \ + a -= t; \ +} + +void m00000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + + const u32 pre_cd = c_rev ^ d_rev; + + MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); + MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x pre_d = d_rev; + const u32x pre_a = a_rev - w0; + const u32x pre_b = b_rev - (pre_a ^ pre_cd); + const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + + if (MATCHES_NONE_VV (pre_c, c)) continue; + + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + + if (MATCHES_NONE_VV (pre_d, d)) continue; + + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00000_a3-pure.cl b/OpenCL/m00000_a3-pure.cl deleted file mode 100644 index 175c759fd..000000000 --- a/OpenCL/m00000_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00000_a3.cl b/OpenCL/m00000_a3.cl index c775ce0e7..175c759fd 100644 --- a/OpenCL/m00000_a3.cl +++ b/OpenCL/m00000_a3.cl @@ -13,109 +13,39 @@ #include "inc_simd.cl" #include "inc_hash_md5.cl" -#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= f (b, c, d); \ - a -= x; \ - a -= t; \ -} - -#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= x; \ - a -= t; \ -} - -void m00000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; + u32x w[64] = { 0 }; - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -123,163 +53,35 @@ void m00000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + w[0] = w0; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_ctx_vector_t ctx; - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_init_vector (&ctx); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + md5_final_vector (&ctx); - COMPARE_M_SIMD (a, d, c, b); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; - - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; - - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; - - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + if (gid >= gid_max) return; /** * digest @@ -294,41 +96,27 @@ void m00000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + const u32 pw_len = pws[gid].pw_len; - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 pre_cd = c_rev ^ d_rev; + u32x w[64] = { 0 }; - MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); - MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -336,318 +124,21 @@ void m00000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x pre_d = d_rev; - const u32x pre_a = a_rev - w0; - const u32x pre_b = b_rev - (pre_a ^ pre_cd); - const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + w[0] = w0; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_ctx_vector_t ctx; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_init_vector (&ctx); - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + md5_final_vector (&ctx); - if (MATCHES_NONE_VV (pre_c, c)) continue; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); - - if (MATCHES_NONE_VV (pre_d, d)) continue; - - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00010_a0-optimized.cl b/OpenCL/m00010_a0-optimized.cl new file mode 100644 index 000000000..ebc59fa6f --- /dev/null +++ b/OpenCL/m00010_a0-optimized.cl @@ -0,0 +1,444 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00010_a0-pure.cl b/OpenCL/m00010_a0-pure.cl deleted file mode 100644 index f353def68..000000000 --- a/OpenCL/m00010_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00010_a0.cl b/OpenCL/m00010_a0.cl index ebc59fa6f..f353def68 100644 --- a/OpenCL/m00010_a0.cl +++ b/OpenCL/m00010_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,273 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - /** - * append salt - */ + md5_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_update (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + md5_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + md5_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -291,154 +106,59 @@ __kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - /** - * append salt - */ + md5_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_update (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + md5_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + md5_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00010_a1-optimized.cl b/OpenCL/m00010_a1-optimized.cl new file mode 100644 index 000000000..af69d9ad5 --- /dev/null +++ b/OpenCL/m00010_a1-optimized.cl @@ -0,0 +1,562 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00010_a1-pure.cl b/OpenCL/m00010_a1-pure.cl deleted file mode 100644 index 7a18b79d6..000000000 --- a/OpenCL/m00010_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00010_a1.cl b/OpenCL/m00010_a1.cl index af69d9ad5..7a18b79d6 100644 --- a/OpenCL/m00010_a1.cl +++ b/OpenCL/m00010_a1.cl @@ -3,340 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -349,214 +93,48 @@ __kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00010_a3-optimized.cl b/OpenCL/m00010_a3-optimized.cl new file mode 100644 index 000000000..2dbb94d9a --- /dev/null +++ b/OpenCL/m00010_a3-optimized.cl @@ -0,0 +1,701 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= f (b, c, d); \ + a -= x; \ + a -= t; \ +} + +#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= x; \ + a -= t; \ +} + +void m00010m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= salt_buf0[0]; + w[ 1] |= salt_buf0[1]; + w[ 2] |= salt_buf0[2]; + w[ 3] |= salt_buf0[3]; + w[ 4] |= salt_buf1[0]; + w[ 5] |= salt_buf1[1]; + w[ 6] |= salt_buf1[2]; + w[ 7] |= salt_buf1[3]; + w[ 8] |= salt_buf2[0]; + w[ 9] |= salt_buf2[1]; + w[10] |= salt_buf2[2]; + w[11] |= salt_buf2[3]; + w[12] |= salt_buf3[0]; + w[13] |= salt_buf3[1]; + w[14] = pw_salt_len * 8; + w[15] = 0; + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00010s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + + const u32 pre_cd = c_rev ^ d_rev; + + MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); + MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x pre_d = d_rev; + const u32x pre_a = a_rev - w0; + const u32x pre_b = b_rev - (pre_a ^ pre_cd); + const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + + if (MATCHES_NONE_VV (pre_c, c)) continue; + + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + + if (MATCHES_NONE_VV (pre_d, d)) continue; + + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00010_a3-pure.cl b/OpenCL/m00010_a3-pure.cl deleted file mode 100644 index 304bad19d..000000000 --- a/OpenCL/m00010_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00010_a3.cl b/OpenCL/m00010_a3.cl index 2dbb94d9a..304bad19d 100644 --- a/OpenCL/m00010_a3.cl +++ b/OpenCL/m00010_a3.cl @@ -11,159 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= f (b, c, d); \ - a -= x; \ - a -= t; \ -} - -#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= x; \ - a -= t; \ -} - -void m00010m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = pw_len + salt_len; - - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); - - w[ 0] |= salt_buf0[0]; - w[ 1] |= salt_buf0[1]; - w[ 2] |= salt_buf0[2]; - w[ 3] |= salt_buf0[3]; - w[ 4] |= salt_buf1[0]; - w[ 5] |= salt_buf1[1]; - w[ 6] |= salt_buf1[2]; - w[ 7] |= salt_buf1[3]; - w[ 8] |= salt_buf2[0]; - w[ 9] |= salt_buf2[1]; - w[10] |= salt_buf2[2]; - w[11] |= salt_buf2[3]; - w[12] |= salt_buf3[0]; - w[13] |= salt_buf3[1]; - w[14] = pw_salt_len * 8; - w[15] = 0; + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; + u32x w[64] = { 0 }; - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -171,163 +66,37 @@ void m00010m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + w[0] = w0; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_ctx_vector_t ctx; - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_init_vector (&ctx); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + md5_update_vector (&ctx, s, salt_len); - COMPARE_M_SIMD (a, d, c, b); + md5_final_vector (&ctx); + + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00010s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; - - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; - - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; - - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + if (gid >= gid_max) return; /** * digest @@ -342,41 +111,40 @@ void m00010s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + const u32 pw_len = pws[gid].pw_len; - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 pre_cd = c_rev ^ d_rev; + u32x w[64] = { 0 }; - MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); - MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -384,318 +152,23 @@ void m00010s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x pre_d = d_rev; - const u32x pre_a = a_rev - w0; - const u32x pre_b = b_rev - (pre_a ^ pre_cd); - const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + w[0] = w0; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_ctx_vector_t ctx; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_init_vector (&ctx); - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + md5_update_vector (&ctx, s, salt_len); - if (MATCHES_NONE_VV (pre_c, c)) continue; + md5_final_vector (&ctx); - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - if (MATCHES_NONE_VV (pre_d, d)) continue; - - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00010s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00020_a0-optimized.cl b/OpenCL/m00020_a0-optimized.cl new file mode 100644 index 000000000..bffca539f --- /dev/null +++ b/OpenCL/m00020_a0-optimized.cl @@ -0,0 +1,404 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00020_a0-pure.cl b/OpenCL/m00020_a0-pure.cl deleted file mode 100644 index ec7f7187d..000000000 --- a/OpenCL/m00020_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00020_a0.cl b/OpenCL/m00020_a0.cl index bffca539f..ec7f7187d 100644 --- a/OpenCL/m00020_a0.cl +++ b/OpenCL/m00020_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,253 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - /** - * prepend salt - */ + md5_update (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + md5_final (&ctx); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -271,134 +95,48 @@ __kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - /** - * prepend salt - */ + md5_update (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + md5_final (&ctx); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00020_a1-optimized.cl b/OpenCL/m00020_a1-optimized.cl new file mode 100644 index 000000000..633944ef4 --- /dev/null +++ b/OpenCL/m00020_a1-optimized.cl @@ -0,0 +1,518 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00020_a1-pure.cl b/OpenCL/m00020_a1-pure.cl deleted file mode 100644 index 7c3d02421..000000000 --- a/OpenCL/m00020_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00020_a1.cl b/OpenCL/m00020_a1.cl index 633944ef4..7c3d02421 100644 --- a/OpenCL/m00020_a1.cl +++ b/OpenCL/m00020_a1.cl @@ -3,318 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + md5_ctx_t ctx0; - if (gid >= gid_max) return; + md5_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -327,192 +80,35 @@ __kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00020_a3-optimized.cl b/OpenCL/m00020_a3-optimized.cl new file mode 100644 index 000000000..002119f5e --- /dev/null +++ b/OpenCL/m00020_a3-optimized.cl @@ -0,0 +1,668 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00020m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00020s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00020_a3-pure.cl b/OpenCL/m00020_a3-pure.cl deleted file mode 100644 index fb46e8374..000000000 --- a/OpenCL/m00020_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00020_a3.cl b/OpenCL/m00020_a3.cl index 002119f5e..fb46e8374 100644 --- a/OpenCL/m00020_a3.cl +++ b/OpenCL/m00020_a3.cl @@ -11,194 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -void m00020m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00020_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + md5_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * md5 - */ + md5_ctx_vector_t ctx; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + md5_final_vector (&ctx); - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00020s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00020_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -213,456 +102,55 @@ void m00020s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + md5_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * md5 - */ + md5_ctx_vector_t ctx; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + md5_update_vector (&ctx, w, pw_len); - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + md5_final_vector (&ctx); - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00020_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00020_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00020_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00020_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00020_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00020_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00020s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00030_a0-optimized.cl b/OpenCL/m00030_a0-optimized.cl new file mode 100644 index 000000000..6820048ac --- /dev/null +++ b/OpenCL/m00030_a0-optimized.cl @@ -0,0 +1,454 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00030_a0-pure.cl b/OpenCL/m00030_a0-pure.cl deleted file mode 100644 index 81fe98a62..000000000 --- a/OpenCL/m00030_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update_utf16le (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update_utf16le (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00030_a0.cl b/OpenCL/m00030_a0.cl index 6820048ac..81fe98a62 100644 --- a/OpenCL/m00030_a0.cl +++ b/OpenCL/m00030_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,278 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md5_init (&ctx); - const u32x out_len2 = out_len * 2; + md5_update_utf16le (&ctx, w, pw_len); - /** - * append salt - */ + md5_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -296,159 +106,59 @@ __kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md5_init (&ctx); - const u32x out_len2 = out_len * 2; + md5_update_utf16le (&ctx, w, pw_len); - /** - * append salt - */ + md5_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00030_a1-optimized.cl b/OpenCL/m00030_a1-optimized.cl new file mode 100644 index 000000000..19a3ce1cc --- /dev/null +++ b/OpenCL/m00030_a1-optimized.cl @@ -0,0 +1,572 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00030_a1-pure.cl b/OpenCL/m00030_a1-pure.cl deleted file mode 100644 index 2160c48e9..000000000 --- a/OpenCL/m00030_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00030_a1.cl b/OpenCL/m00030_a1.cl index 19a3ce1cc..2160c48e9 100644 --- a/OpenCL/m00030_a1.cl +++ b/OpenCL/m00030_a1.cl @@ -3,345 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -354,219 +93,48 @@ __kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00030_a3-optimized.cl b/OpenCL/m00030_a3-optimized.cl new file mode 100644 index 000000000..179c81698 --- /dev/null +++ b/OpenCL/m00030_a3-optimized.cl @@ -0,0 +1,701 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= f (b, c, d); \ + a -= x; \ + a -= t; \ +} + +#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ +{ \ + a -= b; \ + a = rotr32_S (a, s); \ + a -= x; \ + a -= t; \ +} + +void m00030m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= salt_buf0[0]; + w[ 1] |= salt_buf0[1]; + w[ 2] |= salt_buf0[2]; + w[ 3] |= salt_buf0[3]; + w[ 4] |= salt_buf1[0]; + w[ 5] |= salt_buf1[1]; + w[ 6] |= salt_buf1[2]; + w[ 7] |= salt_buf1[3]; + w[ 8] |= salt_buf2[0]; + w[ 9] |= salt_buf2[1]; + w[10] |= salt_buf2[2]; + w[11] |= salt_buf2[3]; + w[12] |= salt_buf3[0]; + w[13] |= salt_buf3[1]; + w[14] = pw_salt_len * 8; + w[15] = 0; + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00030s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0u + MD5C00; + const u32 F_w1c01 = w[ 1] + MD5C01; + const u32 F_w2c02 = w[ 2] + MD5C02; + const u32 F_w3c03 = w[ 3] + MD5C03; + const u32 F_w4c04 = w[ 4] + MD5C04; + const u32 F_w5c05 = w[ 5] + MD5C05; + const u32 F_w6c06 = w[ 6] + MD5C06; + const u32 F_w7c07 = w[ 7] + MD5C07; + const u32 F_w8c08 = w[ 8] + MD5C08; + const u32 F_w9c09 = w[ 9] + MD5C09; + const u32 F_wac0a = w[10] + MD5C0a; + const u32 F_wbc0b = w[11] + MD5C0b; + const u32 F_wcc0c = w[12] + MD5C0c; + const u32 F_wdc0d = w[13] + MD5C0d; + const u32 F_wec0e = w[14] + MD5C0e; + const u32 F_wfc0f = w[15] + MD5C0f; + + const u32 G_w1c10 = w[ 1] + MD5C10; + const u32 G_w6c11 = w[ 6] + MD5C11; + const u32 G_wbc12 = w[11] + MD5C12; + const u32 G_w0c13 = 0u + MD5C13; + const u32 G_w5c14 = w[ 5] + MD5C14; + const u32 G_wac15 = w[10] + MD5C15; + const u32 G_wfc16 = w[15] + MD5C16; + const u32 G_w4c17 = w[ 4] + MD5C17; + const u32 G_w9c18 = w[ 9] + MD5C18; + const u32 G_wec19 = w[14] + MD5C19; + const u32 G_w3c1a = w[ 3] + MD5C1a; + const u32 G_w8c1b = w[ 8] + MD5C1b; + const u32 G_wdc1c = w[13] + MD5C1c; + const u32 G_w2c1d = w[ 2] + MD5C1d; + const u32 G_w7c1e = w[ 7] + MD5C1e; + const u32 G_wcc1f = w[12] + MD5C1f; + + const u32 H_w5c20 = w[ 5] + MD5C20; + const u32 H_w8c21 = w[ 8] + MD5C21; + const u32 H_wbc22 = w[11] + MD5C22; + const u32 H_wec23 = w[14] + MD5C23; + const u32 H_w1c24 = w[ 1] + MD5C24; + const u32 H_w4c25 = w[ 4] + MD5C25; + const u32 H_w7c26 = w[ 7] + MD5C26; + const u32 H_wac27 = w[10] + MD5C27; + const u32 H_wdc28 = w[13] + MD5C28; + const u32 H_w0c29 = 0u + MD5C29; + const u32 H_w3c2a = w[ 3] + MD5C2a; + const u32 H_w6c2b = w[ 6] + MD5C2b; + const u32 H_w9c2c = w[ 9] + MD5C2c; + const u32 H_wcc2d = w[12] + MD5C2d; + const u32 H_wfc2e = w[15] + MD5C2e; + const u32 H_w2c2f = w[ 2] + MD5C2f; + + const u32 I_w0c30 = 0u + MD5C30; + const u32 I_w7c31 = w[ 7] + MD5C31; + const u32 I_wec32 = w[14] + MD5C32; + const u32 I_w5c33 = w[ 5] + MD5C33; + const u32 I_wcc34 = w[12] + MD5C34; + const u32 I_w3c35 = w[ 3] + MD5C35; + const u32 I_wac36 = w[10] + MD5C36; + const u32 I_w1c37 = w[ 1] + MD5C37; + const u32 I_w8c38 = w[ 8] + MD5C38; + const u32 I_wfc39 = w[15] + MD5C39; + const u32 I_w6c3a = w[ 6] + MD5C3a; + const u32 I_wdc3b = w[13] + MD5C3b; + const u32 I_w4c3c = w[ 4] + MD5C3c; + const u32 I_wbc3d = w[11] + MD5C3d; + const u32 I_w2c3e = w[ 2] + MD5C3e; + const u32 I_w9c3f = w[ 9] + MD5C3f; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); + MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); + MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); + MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); + MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + + const u32 pre_cd = c_rev ^ d_rev; + + MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); + MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x pre_d = d_rev; + const u32x pre_a = a_rev - w0; + const u32x pre_b = b_rev - (pre_a ^ pre_cd); + const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + + if (MATCHES_NONE_VV (pre_c, c)) continue; + + MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + + if (MATCHES_NONE_VV (pre_d, d)) continue; + + MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00030_a3-pure.cl b/OpenCL/m00030_a3-pure.cl deleted file mode 100644 index 10c2b0c00..000000000 --- a/OpenCL/m00030_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector_utf16le (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector_utf16le (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00030_a3.cl b/OpenCL/m00030_a3.cl index 179c81698..10c2b0c00 100644 --- a/OpenCL/m00030_a3.cl +++ b/OpenCL/m00030_a3.cl @@ -11,159 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -#define MD5_STEP_REV(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= f (b, c, d); \ - a -= x; \ - a -= t; \ -} - -#define MD5_STEP_REV1(f,a,b,c,d,x,t,s) \ -{ \ - a -= b; \ - a = rotr32_S (a, s); \ - a -= x; \ - a -= t; \ -} - -void m00030m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00030_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = pw_len + salt_len; - - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); - - w[ 0] |= salt_buf0[0]; - w[ 1] |= salt_buf0[1]; - w[ 2] |= salt_buf0[2]; - w[ 3] |= salt_buf0[3]; - w[ 4] |= salt_buf1[0]; - w[ 5] |= salt_buf1[1]; - w[ 6] |= salt_buf1[2]; - w[ 7] |= salt_buf1[3]; - w[ 8] |= salt_buf2[0]; - w[ 9] |= salt_buf2[1]; - w[10] |= salt_buf2[2]; - w[11] |= salt_buf2[3]; - w[12] |= salt_buf3[0]; - w[13] |= salt_buf3[1]; - w[14] = pw_salt_len * 8; - w[15] = 0; + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; + u32x w[64] = { 0 }; - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -171,163 +66,37 @@ void m00030m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + w[0] = w0; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_ctx_vector_t ctx; - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_init_vector (&ctx); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); + md5_update_vector_utf16le (&ctx, w, pw_len); - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); + md5_update_vector (&ctx, s, salt_len); - COMPARE_M_SIMD (a, d, c, b); + md5_final_vector (&ctx); + + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00030s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00030_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 F_w0c00 = 0u + MD5C00; - const u32 F_w1c01 = w[ 1] + MD5C01; - const u32 F_w2c02 = w[ 2] + MD5C02; - const u32 F_w3c03 = w[ 3] + MD5C03; - const u32 F_w4c04 = w[ 4] + MD5C04; - const u32 F_w5c05 = w[ 5] + MD5C05; - const u32 F_w6c06 = w[ 6] + MD5C06; - const u32 F_w7c07 = w[ 7] + MD5C07; - const u32 F_w8c08 = w[ 8] + MD5C08; - const u32 F_w9c09 = w[ 9] + MD5C09; - const u32 F_wac0a = w[10] + MD5C0a; - const u32 F_wbc0b = w[11] + MD5C0b; - const u32 F_wcc0c = w[12] + MD5C0c; - const u32 F_wdc0d = w[13] + MD5C0d; - const u32 F_wec0e = w[14] + MD5C0e; - const u32 F_wfc0f = w[15] + MD5C0f; - - const u32 G_w1c10 = w[ 1] + MD5C10; - const u32 G_w6c11 = w[ 6] + MD5C11; - const u32 G_wbc12 = w[11] + MD5C12; - const u32 G_w0c13 = 0u + MD5C13; - const u32 G_w5c14 = w[ 5] + MD5C14; - const u32 G_wac15 = w[10] + MD5C15; - const u32 G_wfc16 = w[15] + MD5C16; - const u32 G_w4c17 = w[ 4] + MD5C17; - const u32 G_w9c18 = w[ 9] + MD5C18; - const u32 G_wec19 = w[14] + MD5C19; - const u32 G_w3c1a = w[ 3] + MD5C1a; - const u32 G_w8c1b = w[ 8] + MD5C1b; - const u32 G_wdc1c = w[13] + MD5C1c; - const u32 G_w2c1d = w[ 2] + MD5C1d; - const u32 G_w7c1e = w[ 7] + MD5C1e; - const u32 G_wcc1f = w[12] + MD5C1f; - - const u32 H_w5c20 = w[ 5] + MD5C20; - const u32 H_w8c21 = w[ 8] + MD5C21; - const u32 H_wbc22 = w[11] + MD5C22; - const u32 H_wec23 = w[14] + MD5C23; - const u32 H_w1c24 = w[ 1] + MD5C24; - const u32 H_w4c25 = w[ 4] + MD5C25; - const u32 H_w7c26 = w[ 7] + MD5C26; - const u32 H_wac27 = w[10] + MD5C27; - const u32 H_wdc28 = w[13] + MD5C28; - const u32 H_w0c29 = 0u + MD5C29; - const u32 H_w3c2a = w[ 3] + MD5C2a; - const u32 H_w6c2b = w[ 6] + MD5C2b; - const u32 H_w9c2c = w[ 9] + MD5C2c; - const u32 H_wcc2d = w[12] + MD5C2d; - const u32 H_wfc2e = w[15] + MD5C2e; - const u32 H_w2c2f = w[ 2] + MD5C2f; - - const u32 I_w0c30 = 0u + MD5C30; - const u32 I_w7c31 = w[ 7] + MD5C31; - const u32 I_wec32 = w[14] + MD5C32; - const u32 I_w5c33 = w[ 5] + MD5C33; - const u32 I_wcc34 = w[12] + MD5C34; - const u32 I_w3c35 = w[ 3] + MD5C35; - const u32 I_wac36 = w[10] + MD5C36; - const u32 I_w1c37 = w[ 1] + MD5C37; - const u32 I_w8c38 = w[ 8] + MD5C38; - const u32 I_wfc39 = w[15] + MD5C39; - const u32 I_w6c3a = w[ 6] + MD5C3a; - const u32 I_wdc3b = w[13] + MD5C3b; - const u32 I_w4c3c = w[ 4] + MD5C3c; - const u32 I_wbc3d = w[11] + MD5C3d; - const u32 I_w2c3e = w[ 2] + MD5C3e; - const u32 I_w9c3f = w[ 9] + MD5C3f; + if (gid >= gid_max) return; /** * digest @@ -342,41 +111,40 @@ void m00030s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + const u32 pw_len = pws[gid].pw_len; - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 9], MD5C3f, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 2], MD5C3e, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[11], MD5C3d, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 4], MD5C3c, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[13], MD5C3b, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD5C3a, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[15], MD5C39, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[ 8], MD5C38, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 1], MD5C37, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[10], MD5C36, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 3], MD5C35, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, w[12], MD5C34, MD5S30); - MD5_STEP_REV (MD5_I_S, b_rev, c_rev, d_rev, a_rev, w[ 5], MD5C33, MD5S33); - MD5_STEP_REV (MD5_I_S, c_rev, d_rev, a_rev, b_rev, w[14], MD5C32, MD5S32); - MD5_STEP_REV (MD5_I_S, d_rev, a_rev, b_rev, c_rev, w[ 7], MD5C31, MD5S31); - MD5_STEP_REV (MD5_I_S, a_rev, b_rev, c_rev, d_rev, 0, MD5C30, MD5S30); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 pre_cd = c_rev ^ d_rev; + u32x w[64] = { 0 }; - MD5_STEP_REV1(MD5_H_S, b_rev, c_rev, d_rev, a_rev, w[ 2], MD5C2f, MD5S23); - MD5_STEP_REV1(MD5_H_S, c_rev, d_rev, a_rev, b_rev, w[15], MD5C2e, MD5S22); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -384,318 +152,23 @@ void m00030s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x pre_d = d_rev; - const u32x pre_a = a_rev - w0; - const u32x pre_b = b_rev - (pre_a ^ pre_cd); - const u32x pre_c = c_rev - (pre_a ^ pre_b ^ pre_d); + w[0] = w0; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_ctx_vector_t ctx; - MD5_STEP (MD5_Fo, a, b, c, d, w0, F_w0c00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w1c01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w2c02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w3c03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w4c04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w5c05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_w6c06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_w7c07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_w8c08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_w9c09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wac0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wbc0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, F_wcc0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, F_wdc0d, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, F_wec0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, F_wfc0f, MD5S03); + md5_init_vector (&ctx); - MD5_STEP0(MD5_Go, a, b, c, d, G_w1c10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w6c11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wbc12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0, G_w0c13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w5c14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wac15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_wfc16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w4c17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_w9c18, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_wec19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w3c1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_w8c1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, G_wdc1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, G_w2c1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, G_w7c1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, G_wcc1f, MD5S13); + md5_update_vector_utf16le (&ctx, w, pw_len); - MD5_STEP0(MD5_H , a, b, c, d, H_w5c20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w8c21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_wbc22, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wec23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w1c24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_w4c25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w7c26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_wac27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_wdc28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0, H_w0c29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, H_w3c2a, MD5S22); + md5_update_vector (&ctx, s, salt_len); - if (MATCHES_NONE_VV (pre_c, c)) continue; + md5_final_vector (&ctx); - MD5_STEP0(MD5_H , b, c, d, a, H_w6c2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, H_w9c2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, H_wcc2d, MD5S21); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - if (MATCHES_NONE_VV (pre_d, d)) continue; - - MD5_STEP0(MD5_H , c, d, a, b, H_wfc2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, H_w2c2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0, I_w0c30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w7c31, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wec32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w5c33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_wcc34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_w3c35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_wac36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w1c37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w8c38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wfc39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w6c3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_wdc3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, I_w4c3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, I_wbc3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, I_w2c3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, I_w9c3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00030_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00030_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00030_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00030_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00030_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00030_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00030s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00040_a0-optimized.cl b/OpenCL/m00040_a0-optimized.cl new file mode 100644 index 000000000..b58f59785 --- /dev/null +++ b/OpenCL/m00040_a0-optimized.cl @@ -0,0 +1,414 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00040_a0-pure.cl b/OpenCL/m00040_a0-pure.cl deleted file mode 100644 index bf4aa3ff9..000000000 --- a/OpenCL/m00040_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update_utf16le (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update_utf16le (&ctx, w, pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00040_a0.cl b/OpenCL/m00040_a0.cl index b58f59785..bf4aa3ff9 100644 --- a/OpenCL/m00040_a0.cl +++ b/OpenCL/m00040_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,258 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md5_update_utf16le (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + md5_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -276,139 +95,48 @@ __kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md5_update_utf16le (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + md5_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00040_a1-optimized.cl b/OpenCL/m00040_a1-optimized.cl new file mode 100644 index 000000000..2d6c1f21e --- /dev/null +++ b/OpenCL/m00040_a1-optimized.cl @@ -0,0 +1,528 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00040_a1-pure.cl b/OpenCL/m00040_a1-pure.cl deleted file mode 100644 index f539b5ee6..000000000 --- a/OpenCL/m00040_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00040_a1.cl b/OpenCL/m00040_a1.cl index 2d6c1f21e..f539b5ee6 100644 --- a/OpenCL/m00040_a1.cl +++ b/OpenCL/m00040_a1.cl @@ -3,323 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + md5_ctx_t ctx0; - if (gid >= gid_max) return; + md5_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -332,197 +80,35 @@ __kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + md5_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00040_a3-optimized.cl b/OpenCL/m00040_a3-optimized.cl new file mode 100644 index 000000000..8017e0ca9 --- /dev/null +++ b/OpenCL/m00040_a3-optimized.cl @@ -0,0 +1,668 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00040m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00040s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00040_a3-pure.cl b/OpenCL/m00040_a3-pure.cl deleted file mode 100644 index be5995613..000000000 --- a/OpenCL/m00040_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_utf16le (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_utf16le (&ctx, w, pw_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00040_a3.cl b/OpenCL/m00040_a3.cl index 8017e0ca9..be5995613 100644 --- a/OpenCL/m00040_a3.cl +++ b/OpenCL/m00040_a3.cl @@ -11,194 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -void m00040m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00040_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + md5_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * md5 - */ + md5_ctx_vector_t ctx; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + md5_update_vector_utf16le (&ctx, w, pw_len); - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + md5_final_vector (&ctx); - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00040s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00040_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -213,456 +102,55 @@ void m00040s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + md5_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * md5 - */ + md5_ctx_vector_t ctx; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + md5_update_vector_utf16le (&ctx, w, pw_len); - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + md5_final_vector (&ctx); - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00040_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00040_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00040_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00040_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00040_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00040_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00040s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00050_a0-optimized.cl b/OpenCL/m00050_a0-optimized.cl new file mode 100644 index 000000000..6df3d2fd3 --- /dev/null +++ b/OpenCL/m00050_a0-optimized.cl @@ -0,0 +1,436 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * pads + */ + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = (64 + salt_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * pads + */ + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = (64 + salt_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00050_a0-pure.cl b/OpenCL/m00050_a0-pure.cl deleted file mode 100644 index 263ef488f..000000000 --- a/OpenCL/m00050_a0-pure.cl +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_hmac_ctx_t ctx; - - md5_hmac_init (&ctx, w, pw_len); - - md5_hmac_update (&ctx, s, salt_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_hmac_ctx_t ctx; - - md5_hmac_init (&ctx, w, pw_len); - - md5_hmac_update (&ctx, s, salt_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00050_a0.cl b/OpenCL/m00050_a0.cl index 6df3d2fd3..263ef488f 100644 --- a/OpenCL/m00050_a0.cl +++ b/OpenCL/m00050_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,362 +12,86 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_hmac_ctx_t ctx; - /** - * pads - */ + md5_hmac_init (&ctx, w, pw_len); - u32x ipad[4]; - u32x opad[4]; + md5_hmac_update (&ctx, s, salt_len); - hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + md5_hmac_final (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = (64 + salt_len) * 8; - w3[3] = 0; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[4]; - - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -380,57 +104,57 @@ __kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_hmac_ctx_t ctx; - /** - * pads - */ + md5_hmac_init (&ctx, w, pw_len); - u32x ipad[4]; - u32x opad[4]; + md5_hmac_update (&ctx, s, salt_len); - hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + md5_hmac_final (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = (64 + salt_len) * 8; - w3[3] = 0; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[4]; - - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00050_a1-optimized.cl b/OpenCL/m00050_a1-optimized.cl new file mode 100644 index 000000000..d21e276f4 --- /dev/null +++ b/OpenCL/m00050_a1-optimized.cl @@ -0,0 +1,554 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * pads + */ + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = (64 + salt_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * pads + */ + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = (64 + salt_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00050_a1-pure.cl b/OpenCL/m00050_a1-pure.cl deleted file mode 100644 index 7d75b6e4f..000000000 --- a/OpenCL/m00050_a1-pure.cl +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = combs_buf[il_pos].i[idx]; - } - - switch_buffer_by_offset_1x64_le_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - md5_hmac_ctx_vector_t ctx; - - md5_hmac_init_vector (&ctx, c, pw_len + comb_len); - - md5_hmac_update_vector (&ctx, s, salt_len); - - md5_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = combs_buf[il_pos].i[idx]; - } - - switch_buffer_by_offset_1x64_le_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - md5_hmac_ctx_vector_t ctx; - - md5_hmac_init_vector (&ctx, c, pw_len + comb_len); - - md5_hmac_update_vector (&ctx, s, salt_len); - - md5_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00050_a1.cl b/OpenCL/m00050_a1.cl index d21e276f4..7d75b6e4f 100644 --- a/OpenCL/m00050_a1.cl +++ b/OpenCL/m00050_a1.cl @@ -3,429 +3,113 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = combs_buf[il_pos].i[idx]; } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_le_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - /** - * pads - */ + md5_hmac_ctx_vector_t ctx; - u32x ipad[4]; - u32x opad[4]; + md5_hmac_init_vector (&ctx, c, pw_len + comb_len); - hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + md5_hmac_update_vector (&ctx, s, salt_len); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = (64 + salt_len) * 8; - w3[3] = 0; + md5_hmac_final_vector (&ctx); - u32x digest[4]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -438,117 +122,77 @@ __kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = combs_buf[il_pos].i[idx]; } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_le_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - /** - * pads - */ + md5_hmac_ctx_vector_t ctx; - u32x ipad[4]; - u32x opad[4]; + md5_hmac_init_vector (&ctx, c, pw_len + comb_len); - hmac_md5_pad (w0, w1, w2, w3, ipad, opad); + md5_hmac_update_vector (&ctx, s, salt_len); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = (64 + salt_len) * 8; - w3[3] = 0; + md5_hmac_final_vector (&ctx); - u32x digest[4]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00050_a3-optimized.cl b/OpenCL/m00050_a3-optimized.cl new file mode 100644 index 000000000..91383388c --- /dev/null +++ b/OpenCL/m00050_a3-optimized.cl @@ -0,0 +1,700 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +void m00050m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = (64 + salt_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +void m00050s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = (64 + salt_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00050_a3-pure.cl b/OpenCL/m00050_a3-pure.cl deleted file mode 100644 index 09f223a60..000000000 --- a/OpenCL/m00050_a3-pure.cl +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_hmac_ctx_vector_t ctx; - - md5_hmac_init_vector (&ctx, w, pw_len); - - md5_hmac_update_vector (&ctx, s, salt_len); - - md5_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_hmac_ctx_vector_t ctx; - - md5_hmac_init_vector (&ctx, w, pw_len); - - md5_hmac_update_vector (&ctx, s, salt_len); - - md5_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00050_a3.cl b/OpenCL/m00050_a3.cl index 91383388c..09f223a60 100644 --- a/OpenCL/m00050_a3.cl +++ b/OpenCL/m00050_a3.cl @@ -11,331 +11,90 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -void m00050m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00050_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md5_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[4]; - u32x opad[4]; + md5_hmac_update_vector (&ctx, s, salt_len); - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + md5_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = (64 + salt_len) * 8; - w3_t[3] = 0; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[4]; - - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00050s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00050_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + if (gid >= gid_max) return; /** * digest @@ -349,352 +108,63 @@ void m00050s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md5_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[4]; - u32x opad[4]; + md5_hmac_update_vector (&ctx, s, salt_len); - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + md5_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = (64 + salt_len) * 8; - w3_t[3] = 0; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[4]; - - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00050_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00050_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00050_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00050_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00050_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00050_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00050s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00060_a0-optimized.cl b/OpenCL/m00060_a0-optimized.cl new file mode 100644 index 000000000..132c4d62d --- /dev/null +++ b/OpenCL/m00060_a0-optimized.cl @@ -0,0 +1,484 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = w0[0]; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = (64 + out_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = w0[0]; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = (64 + out_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00060_a0-pure.cl b/OpenCL/m00060_a0-pure.cl deleted file mode 100644 index ea4b96827..000000000 --- a/OpenCL/m00060_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_t ctx0; - - md5_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_hmac_ctx_t ctx = ctx0; - - md5_hmac_update (&ctx, w, pw_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_t ctx0; - - md5_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_hmac_ctx_t ctx = ctx0; - - md5_hmac_update (&ctx, w, pw_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00060_a0.cl b/OpenCL/m00060_a0.cl index 132c4d62d..ea4b96827 100644 --- a/OpenCL/m00060_a0.cl +++ b/OpenCL/m00060_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,417 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - u32x ipad[4]; - u32x opad[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + md5_hmac_ctx_t ctx0; + + md5_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_hmac_update (&ctx, w, pw_len); - w0_t[0] = w0[0]; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = (64 + out_len) * 8; - w3_t[3] = 0; + md5_hmac_final (&ctx); - u32x digest[4]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[4]; - u32x opad[4]; - - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -435,50 +106,59 @@ __kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_hmac_ctx_t ctx0; + + md5_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_hmac_update (&ctx, w, pw_len); - w0_t[0] = w0[0]; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = (64 + out_len) * 8; - w3_t[3] = 0; + md5_hmac_final (&ctx); - u32x digest[4]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00060_a1-optimized.cl b/OpenCL/m00060_a1-optimized.cl new file mode 100644 index 000000000..6db185ad2 --- /dev/null +++ b/OpenCL/m00060_a1-optimized.cl @@ -0,0 +1,564 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = (64 + pw_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = (64 + pw_len) * 8; + w3[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00060_a1-pure.cl b/OpenCL/m00060_a1-pure.cl deleted file mode 100644 index a0e4f0a23..000000000 --- a/OpenCL/m00060_a1-pure.cl +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_t ctx0; - - md5_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = combs_buf[il_pos].i[idx]; - } - - switch_buffer_by_offset_1x64_le_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - md5_hmac_ctx_t ctx = ctx0; - - md5_hmac_update (&ctx, c, pw_len + comb_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_t ctx0; - - md5_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = combs_buf[il_pos].i[idx]; - } - - switch_buffer_by_offset_1x64_le_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - md5_hmac_ctx_t ctx = ctx0; - - md5_hmac_update (&ctx, c, pw_len + comb_len); - - md5_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00060_a1.cl b/OpenCL/m00060_a1.cl index 6db185ad2..a0e4f0a23 100644 --- a/OpenCL/m00060_a1.cl +++ b/OpenCL/m00060_a1.cl @@ -3,465 +3,115 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - u32x ipad[4]; - u32x opad[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + md5_hmac_ctx_t ctx0; + + md5_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = combs_buf[il_pos].i[idx]; } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_le_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = (64 + pw_len) * 8; - w3[3] = 0; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - u32x digest[4]; + md5_hmac_ctx_t ctx = ctx0; - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + md5_hmac_update (&ctx, c, pw_len + comb_len); - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + md5_hmac_final (&ctx); + + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; + + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[4]; - u32x opad[4]; - - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -474,91 +124,79 @@ __kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_hmac_ctx_t ctx0; + + md5_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = combs_buf[il_pos].i[idx]; } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_le_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = (64 + pw_len) * 8; - w3[3] = 0; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - u32x digest[4]; + md5_hmac_ctx_t ctx = ctx0; - hmac_md5_run (w0, w1, w2, w3, ipad, opad, digest); + md5_hmac_update (&ctx, c, pw_len + comb_len); - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + md5_hmac_final (&ctx); + + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; + + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00060_a3-optimized.cl b/OpenCL/m00060_a3-optimized.cl new file mode 100644 index 000000000..d31c1a0de --- /dev/null +++ b/OpenCL/m00060_a3-optimized.cl @@ -0,0 +1,700 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = MD5M_A; + ipad[1] = MD5M_B; + ipad[2] = MD5M_C; + ipad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = MD5M_A; + opad[1] = MD5M_B; + opad[2] = MD5M_C; + opad[3] = MD5M_D; + + md5_transform (w0, w1, w2, w3, opad); +} + +void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + + md5_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = 0x80; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = (64 + 16) * 8; + w3[3] = 0; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + + md5_transform (w0, w1, w2, w3, digest); +} + +void m00060m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = (64 + pw_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +void m00060s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[4]; + u32x opad[4]; + + hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = (64 + pw_len) * 8; + w3_t[3] = 0; + + u32x digest[4]; + + hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + } +} + +__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00060_a3-pure.cl b/OpenCL/m00060_a3-pure.cl deleted file mode 100644 index 1c1d79a29..000000000 --- a/OpenCL/m00060_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_vector_t ctx0; - - md5_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_hmac_ctx_vector_t ctx = ctx0; - - md5_hmac_update_vector (&ctx, w, pw_len); - - md5_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_hmac_ctx_vector_t ctx0; - - md5_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_hmac_ctx_vector_t ctx = ctx0; - - md5_hmac_update_vector (&ctx, w, pw_len); - - md5_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00060_a3.cl b/OpenCL/m00060_a3.cl index d31c1a0de..1c1d79a29 100644 --- a/OpenCL/m00060_a3.cl +++ b/OpenCL/m00060_a3.cl @@ -11,362 +11,92 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -void md5_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void hmac_md5_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = MD5M_A; - ipad[1] = MD5M_B; - ipad[2] = MD5M_C; - ipad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = MD5M_A; - opad[1] = MD5M_B; - opad[2] = MD5M_C; - opad[3] = MD5M_D; - - md5_transform (w0, w1, w2, w3, opad); -} - -void hmac_md5_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[4], u32x opad[4], u32x digest[4]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - - md5_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = 0x80; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = (64 + 16) * 8; - w3[3] = 0; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - - md5_transform (w0, w1, w2, w3, digest); -} - -void m00060m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00060_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32x s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - u32x ipad[4]; - u32x opad[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + md5_hmac_ctx_vector_t ctx0; + + md5_hmac_init_vector (&ctx0, s, salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = (64 + pw_len) * 8; - w3_t[3] = 0; + w[0] = w0; - u32x digest[4]; + md5_hmac_ctx_vector_t ctx = ctx0; - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + md5_hmac_update_vector (&ctx, w, pw_len); - COMPARE_M_SIMD (digest[0], digest[3], digest[2], digest[1]); + md5_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00060s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00060_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[4]; - u32x opad[4]; - - hmac_md5_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + if (gid >= gid_max) return; /** * digest @@ -380,321 +110,65 @@ void m00060s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_hmac_ctx_vector_t ctx0; + + md5_hmac_init_vector (&ctx0, s, salt_len); + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = (64 + pw_len) * 8; - w3_t[3] = 0; + w[0] = w0; - u32x digest[4]; + md5_hmac_ctx_vector_t ctx = ctx0; - hmac_md5_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + md5_hmac_update_vector (&ctx, w, pw_len); - COMPARE_S_SIMD (digest[0], digest[3], digest[2], digest[1]); + md5_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00060_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00060_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00060_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00060_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00060_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00060_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00060s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00100_a0-optimized.cl b/OpenCL/m00100_a0-optimized.cl new file mode 100644 index 000000000..ae2e774a3 --- /dev/null +++ b/OpenCL/m00100_a0-optimized.cl @@ -0,0 +1,396 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00100_a0-pure.cl b/OpenCL/m00100_a0-pure.cl deleted file mode 100644 index a8bd7ec57..000000000 --- a/OpenCL/m00100_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00100_a0.cl b/OpenCL/m00100_a0.cl index ae2e774a3..a8bd7ec57 100644 --- a/OpenCL/m00100_a0.cl +++ b/OpenCL/m00100_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,218 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_init (&ctx); - /** - * sha1 - */ + sha1_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha1_final (&ctx); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -237,160 +92,43 @@ __kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_init (&ctx); - /** - * sha1 - */ + sha1_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha1_final (&ctx); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00100_a1-optimized.cl b/OpenCL/m00100_a1-optimized.cl new file mode 100644 index 000000000..8cd1f7182 --- /dev/null +++ b/OpenCL/m00100_a1-optimized.cl @@ -0,0 +1,510 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00100_a1-pure.cl b/OpenCL/m00100_a1-pure.cl deleted file mode 100644 index 14153d297..000000000 --- a/OpenCL/m00100_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00100_a1.cl b/OpenCL/m00100_a1.cl index 8cd1f7182..14153d297 100644 --- a/OpenCL/m00100_a1.cl +++ b/OpenCL/m00100_a1.cl @@ -3,283 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha1_ctx_t ctx0; - if (gid >= gid_max) return; + sha1_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -293,218 +79,32 @@ __kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00100_a3-optimized.cl b/OpenCL/m00100_a3-optimized.cl new file mode 100644 index 000000000..b1b15b3d4 --- /dev/null +++ b/OpenCL/m00100_a3-optimized.cl @@ -0,0 +1,752 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00100s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + + SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00100_a3-pure.cl b/OpenCL/m00100_a3-pure.cl deleted file mode 100644 index 50ecd137b..000000000 --- a/OpenCL/m00100_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00100_a3.cl b/OpenCL/m00100_a3.cl index b1b15b3d4..50ecd137b 100644 --- a/OpenCL/m00100_a3.cl +++ b/OpenCL/m00100_a3.cl @@ -11,102 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + const u32 pw_len = pws[gid].pw_len; - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -114,237 +53,35 @@ void m00100m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00100s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + if (gid >= gid_max) return; /** * digest @@ -359,16 +96,27 @@ void m00100s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + const u32 pw_len = pws[gid].pw_len; - /** + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -376,377 +124,21 @@ void m00100s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - - SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00100s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00110_a0-optimized.cl b/OpenCL/m00110_a0-optimized.cl new file mode 100644 index 000000000..9464be718 --- /dev/null +++ b/OpenCL/m00110_a0-optimized.cl @@ -0,0 +1,542 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00110_a0-pure.cl b/OpenCL/m00110_a0-pure.cl deleted file mode 100644 index da9c38d03..000000000 --- a/OpenCL/m00110_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_swap (&ctx, w, pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_swap (&ctx, w, pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00110_a0.cl b/OpenCL/m00110_a0.cl index 9464be718..da9c38d03 100644 --- a/OpenCL/m00110_a0.cl +++ b/OpenCL/m00110_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,319 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - /** - * append salt - */ + sha1_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha1_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha1_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha1_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -338,205 +107,58 @@ __kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - /** - * append salt - */ + sha1_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha1_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha1_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha1_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00110_a1-optimized.cl b/OpenCL/m00110_a1-optimized.cl new file mode 100644 index 000000000..c2daccc00 --- /dev/null +++ b/OpenCL/m00110_a1-optimized.cl @@ -0,0 +1,660 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00110_a1-pure.cl b/OpenCL/m00110_a1-pure.cl deleted file mode 100644 index 3d8987f72..000000000 --- a/OpenCL/m00110_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00110_a1.cl b/OpenCL/m00110_a1.cl index c2daccc00..3d8987f72 100644 --- a/OpenCL/m00110_a1.cl +++ b/OpenCL/m00110_a1.cl @@ -3,386 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -396,265 +94,47 @@ __kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00110_a3-optimized.cl b/OpenCL/m00110_a3-optimized.cl new file mode 100644 index 000000000..058fb129a --- /dev/null +++ b/OpenCL/m00110_a3-optimized.cl @@ -0,0 +1,800 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00110m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + const u32 c_76s = rotl32_S ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32 c_77s = rotl32_S ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32 c_78s = rotl32_S ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32 c_79s = rotl32_S ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00110s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + + SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00110_a3-pure.cl b/OpenCL/m00110_a3-pure.cl deleted file mode 100644 index 00418eafc..000000000 --- a/OpenCL/m00110_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_update_vector (&ctx, s, salt_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_update_vector (&ctx, s, salt_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00110_a3.cl b/OpenCL/m00110_a3.cl index 058fb129a..00418eafc 100644 --- a/OpenCL/m00110_a3.cl +++ b/OpenCL/m00110_a3.cl @@ -11,157 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00110m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); - - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = pw_len + salt_len; - - w[15] = pw_salt_len * 8; + if (gid >= gid_max) return; /** * base */ - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - const u32 c_76s = rotl32_S ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32 c_77s = rotl32_S ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32 c_78s = rotl32_S ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32 c_79s = rotl32_S ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + const u32 pw_len = pws[gid].pw_len; - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -169,230 +66,37 @@ void m00110m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_update_vector (&ctx, s, salt_len); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_final_vector (&ctx); - #undef K - #define K SHA1C01 + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00110s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + if (gid >= gid_max) return; /** * digest @@ -407,16 +111,40 @@ void m00110s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + const u32 pw_len = pws[gid].pw_len; - /** + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -424,377 +152,23 @@ void m00110s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_update_vector (&ctx, s, salt_len); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_final_vector (&ctx); - #undef K - #define K SHA1C01 + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - - SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00110s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00120_a0-optimized.cl b/OpenCL/m00120_a0-optimized.cl new file mode 100644 index 000000000..36f523860 --- /dev/null +++ b/OpenCL/m00120_a0-optimized.cl @@ -0,0 +1,514 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00120_a0-pure.cl b/OpenCL/m00120_a0-pure.cl deleted file mode 100644 index 6849a85a8..000000000 --- a/OpenCL/m00120_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx = ctx0; - - sha1_update_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx = ctx0; - - sha1_update_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00120_a0.cl b/OpenCL/m00120_a0.cl index 36f523860..6849a85a8 100644 --- a/OpenCL/m00120_a0.cl +++ b/OpenCL/m00120_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,66 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) + { + // todo: add rules engine + + sha1_ctx_t ctx = ctx0; + + sha1_update_swap (&ctx, w, pw_len); + + sha1_final (&ctx); + + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; + + COMPARE_M_SCALAR (r0, r1, r2, r3); + } +} + +__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -84,431 +95,48 @@ __kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; - - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); - - /** - * prepend salt - */ - - const u32x out_salt_len = out_len + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); - } -} - -__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * digest - */ - - const u32 search[4] = + for (int idx = 0; idx < pw_lenv; idx++) { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; + w[idx] = pws[gid].i[idx]; - /** - * reverse - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 e_rev = rotl32_S (search[1], 2u); + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx = ctx0; - /** - * prepend salt - */ + sha1_update_swap (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + sha1_final (&ctx); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00120_a1-optimized.cl b/OpenCL/m00120_a1-optimized.cl new file mode 100644 index 000000000..56e12ed4d --- /dev/null +++ b/OpenCL/m00120_a1-optimized.cl @@ -0,0 +1,628 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00120_a1-pure.cl b/OpenCL/m00120_a1-pure.cl deleted file mode 100644 index 5f71bfefd..000000000 --- a/OpenCL/m00120_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00120_a1.cl b/OpenCL/m00120_a1.cl index 56e12ed4d..5f71bfefd 100644 --- a/OpenCL/m00120_a1.cl +++ b/OpenCL/m00120_a1.cl @@ -3,73 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) + { + sha1_ctx_t ctx = ctx0; + + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); + + sha1_final (&ctx); + + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; + + COMPARE_M_SCALAR (r0, r1, r2, r3); + } +} + +__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -82,547 +80,35 @@ __kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); - - const u32x pw_len = pw_l_len + pw_r_len; - - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); - } -} - -__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - /** * base */ - const u32 gid = get_global_id (0); + sha1_ctx_t ctx0; - if (gid >= gid_max) return; + sha1_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * reverse - */ - - const u32 e_rev = rotl32_S (search[1], 2u); + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00120_a3-optimized.cl b/OpenCL/m00120_a3-optimized.cl new file mode 100644 index 000000000..2170c91f7 --- /dev/null +++ b/OpenCL/m00120_a3-optimized.cl @@ -0,0 +1,766 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00120m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha1 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00120s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha1 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00120_a3-pure.cl b/OpenCL/m00120_a3-pure.cl deleted file mode 100644 index 178dd369e..000000000 --- a/OpenCL/m00120_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector_from_scalar (&ctx, &ctx0); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector_from_scalar (&ctx, &ctx0); - - sha1_update_vector (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00120_a3.cl b/OpenCL/m00120_a3.cl index 2170c91f7..178dd369e 100644 --- a/OpenCL/m00120_a3.cl +++ b/OpenCL/m00120_a3.cl @@ -11,240 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00120m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00120_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha1_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha1_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha1 - */ + sha1_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha1_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_update_vector (&ctx, w, pw_len); - #undef K - #define K SHA1C00 + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00120s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00120_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -259,508 +102,55 @@ void m00120s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32x w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 pw_salt_len = pw_len + salt_len; + sha1_ctx_t ctx0; - /** - * prepend salt - */ + sha1_init (&ctx0); - const u32 w0l = w0[0]; - - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha1 - */ + sha1_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha1_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_update_vector (&ctx, w, pw_len); - #undef K - #define K SHA1C00 + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00120_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00120_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00120_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00120_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00120_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00120_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00120s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00130_a0-optimized.cl b/OpenCL/m00130_a0-optimized.cl new file mode 100644 index 000000000..bbdb9193c --- /dev/null +++ b/OpenCL/m00130_a0-optimized.cl @@ -0,0 +1,552 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00130_a0-pure.cl b/OpenCL/m00130_a0-pure.cl deleted file mode 100644 index ae3d7372c..000000000 --- a/OpenCL/m00130_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_utf16le_swap (&ctx, w, pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - sha1_update_utf16le_swap (&ctx, w, pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00130_a0.cl b/OpenCL/m00130_a0.cl index bbdb9193c..ae3d7372c 100644 --- a/OpenCL/m00130_a0.cl +++ b/OpenCL/m00130_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,324 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha1_init (&ctx); - const u32x out_len2 = out_len * 2; + sha1_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha1_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha1_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -343,210 +107,58 @@ __kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha1_init (&ctx); - const u32x out_len2 = out_len * 2; + sha1_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha1_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha1_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00130_a1-optimized.cl b/OpenCL/m00130_a1-optimized.cl new file mode 100644 index 000000000..611730441 --- /dev/null +++ b/OpenCL/m00130_a1-optimized.cl @@ -0,0 +1,670 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00130_a1-pure.cl b/OpenCL/m00130_a1-pure.cl deleted file mode 100644 index 2f7f11bd4..000000000 --- a/OpenCL/m00130_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_update (&ctx, s, salt_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00130_a1.cl b/OpenCL/m00130_a1.cl index 611730441..2f7f11bd4 100644 --- a/OpenCL/m00130_a1.cl +++ b/OpenCL/m00130_a1.cl @@ -3,391 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -401,270 +94,47 @@ __kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00130_a3-optimized.cl b/OpenCL/m00130_a3-optimized.cl new file mode 100644 index 000000000..0d5f9bf67 --- /dev/null +++ b/OpenCL/m00130_a3-optimized.cl @@ -0,0 +1,800 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00130m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + const u32 c_76s = rotl32_S ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32 c_77s = rotl32_S ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32 c_78s = rotl32_S ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32 c_79s = rotl32_S ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00130s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + + SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00130_a3-pure.cl b/OpenCL/m00130_a3-pure.cl deleted file mode 100644 index 8024f2233..000000000 --- a/OpenCL/m00130_a3-pure.cl +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector_utf16beN (&ctx, w, pw_len); - - sha1_update_vector (&ctx, s, salt_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - - -__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - sha1_update_vector_utf16beN (&ctx, w, pw_len); - - sha1_update_vector (&ctx, s, salt_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00130_a3.cl b/OpenCL/m00130_a3.cl index 0d5f9bf67..8024f2233 100644 --- a/OpenCL/m00130_a3.cl +++ b/OpenCL/m00130_a3.cl @@ -11,157 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00130m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00130_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); - - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = pw_len + salt_len; - - w[15] = pw_salt_len * 8; + if (gid >= gid_max) return; /** * base */ - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - const u32 c_76s = rotl32_S ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32 c_77s = rotl32_S ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32 c_78s = rotl32_S ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32 c_79s = rotl32_S ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + const u32 pw_len = pws[gid].pw_len; - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -169,230 +66,38 @@ void m00130m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector_utf16beN (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_update_vector (&ctx, s, salt_len); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_final_vector (&ctx); - #undef K - #define K SHA1C01 + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00130s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) + +__kernel void m00130_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + if (gid >= gid_max) return; /** * digest @@ -407,16 +112,40 @@ void m00130s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u) - SHA1C03; + const u32 pw_len = pws[gid].pw_len; - /** + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -424,377 +153,23 @@ void m00130s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx); - #undef K - #define K SHA1C00 + sha1_update_vector_utf16beN (&ctx, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_update_vector (&ctx, s, salt_len); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_final_vector (&ctx); - #undef K - #define K SHA1C01 + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); - - #undef K - #define K SHA1C02 - - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); - - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - - SHA1_STEP_PE (SHA1_F1, a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - SHA1_STEP_PB (SHA1_F1, a, b, c, d, e, 0); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00130_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00130_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00130_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00130_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00130_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00130_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00130s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00140_a0-optimized.cl b/OpenCL/m00140_a0-optimized.cl new file mode 100644 index 000000000..b0e0e98ae --- /dev/null +++ b/OpenCL/m00140_a0-optimized.cl @@ -0,0 +1,512 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00140_a0-pure.cl b/OpenCL/m00140_a0-pure.cl deleted file mode 100644 index 1b8c5e717..000000000 --- a/OpenCL/m00140_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx = ctx0; - - sha1_update_utf16le_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx = ctx0; - - sha1_update_utf16le_swap (&ctx, w, pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00140_a0.cl b/OpenCL/m00140_a0.cl index b0e0e98ae..1b8c5e717 100644 --- a/OpenCL/m00140_a0.cl +++ b/OpenCL/m00140_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,304 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha1_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha1_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -323,190 +96,47 @@ __kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha1_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha1_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00140_a1-optimized.cl b/OpenCL/m00140_a1-optimized.cl new file mode 100644 index 000000000..c1661eb98 --- /dev/null +++ b/OpenCL/m00140_a1-optimized.cl @@ -0,0 +1,626 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00140_a1-pure.cl b/OpenCL/m00140_a1-pure.cl deleted file mode 100644 index 9be7ed08a..000000000 --- a/OpenCL/m00140_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx = ctx0; - - sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00140_a1.cl b/OpenCL/m00140_a1.cl index c1661eb98..9be7ed08a 100644 --- a/OpenCL/m00140_a1.cl +++ b/OpenCL/m00140_a1.cl @@ -3,369 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha1_ctx_t ctx0; - if (gid >= gid_max) return; + sha1_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -379,248 +81,34 @@ __kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha1_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00140_a3-optimized.cl b/OpenCL/m00140_a3-optimized.cl new file mode 100644 index 000000000..a304aa880 --- /dev/null +++ b/OpenCL/m00140_a3-optimized.cl @@ -0,0 +1,766 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00140m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha1 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00140s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha1 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00140_a3-pure.cl b/OpenCL/m00140_a3-pure.cl deleted file mode 100644 index 75d22495b..000000000 --- a/OpenCL/m00140_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector_from_scalar (&ctx, &ctx0); - - sha1_update_vector_utf16beN (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx; - - sha1_init_vector_from_scalar (&ctx, &ctx0); - - sha1_update_vector_utf16beN (&ctx, w, pw_len); - - sha1_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00140_a3.cl b/OpenCL/m00140_a3.cl index a304aa880..75d22495b 100644 --- a/OpenCL/m00140_a3.cl +++ b/OpenCL/m00140_a3.cl @@ -11,240 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00140m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00140_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha1_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha1_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha1 - */ + sha1_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha1_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_update_vector_utf16beN (&ctx, w, pw_len); - #undef K - #define K SHA1C00 + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00140s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00140_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -259,508 +102,55 @@ void m00140s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32x w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 pw_salt_len = pw_len + salt_len; + sha1_ctx_t ctx0; - /** - * prepend salt - */ + sha1_init (&ctx0); - const u32 w0l = w0[0]; - - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha1_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha1 - */ + sha1_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha1_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_update_vector_utf16beN (&ctx, w, pw_len); - #undef K - #define K SHA1C00 + sha1_final_vector (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00140_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00140_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00140_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00140_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00140_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00140_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00140s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00150_a0-optimized.cl b/OpenCL/m00150_a0-optimized.cl new file mode 100644 index 000000000..fcb04e3f2 --- /dev/null +++ b/OpenCL/m00150_a0-optimized.cl @@ -0,0 +1,488 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00150_a0-pure.cl b/OpenCL/m00150_a0-pure.cl deleted file mode 100644 index 957d2a50b..000000000 --- a/OpenCL/m00150_a0-pure.cl +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_hmac_ctx_t ctx; - - sha1_hmac_init_swap (&ctx, w, pw_len); - - sha1_hmac_update (&ctx, s, salt_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_hmac_ctx_t ctx; - - sha1_hmac_init_swap (&ctx, w, pw_len); - - sha1_hmac_update (&ctx, s, salt_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00150_a0.cl b/OpenCL/m00150_a0.cl index fcb04e3f2..957d2a50b 100644 --- a/OpenCL/m00150_a0.cl +++ b/OpenCL/m00150_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,405 +12,86 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha1_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha1_hmac_update (&ctx, s, salt_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_final (&ctx); - hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; - - u32x digest[5]; - - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -423,66 +104,57 @@ __kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha1_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha1_hmac_update (&ctx, s, salt_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_final (&ctx); - hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; - - u32x digest[5]; - - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00150_a1-optimized.cl b/OpenCL/m00150_a1-optimized.cl new file mode 100644 index 000000000..9d3d75690 --- /dev/null +++ b/OpenCL/m00150_a1-optimized.cl @@ -0,0 +1,622 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00150_a1-pure.cl b/OpenCL/m00150_a1-pure.cl deleted file mode 100644 index 15ab612bf..000000000 --- a/OpenCL/m00150_a1-pure.cl +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha1_hmac_ctx_vector_t ctx; - - sha1_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha1_hmac_update_vector (&ctx, s, salt_len); - - sha1_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha1_hmac_ctx_vector_t ctx; - - sha1_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha1_hmac_update_vector (&ctx, s, salt_len); - - sha1_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00150_a1.cl b/OpenCL/m00150_a1.cl index 9d3d75690..15ab612bf 100644 --- a/OpenCL/m00150_a1.cl +++ b/OpenCL/m00150_a1.cl @@ -3,480 +3,113 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha1_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha1_hmac_init_vector (&ctx, c, pw_len + comb_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_update_vector (&ctx, s, salt_len); - hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + sha1_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[5]; - - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -489,134 +122,77 @@ __kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha1_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha1_hmac_init_vector (&ctx, c, pw_len + comb_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_update_vector (&ctx, s, salt_len); - hmac_sha1_pad (w0, w1, w2, w3, ipad, opad); + sha1_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[5]; - - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00150_a3-optimized.cl b/OpenCL/m00150_a3-optimized.cl new file mode 100644 index 000000000..ec392c959 --- /dev/null +++ b/OpenCL/m00150_a3-optimized.cl @@ -0,0 +1,734 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +void m00150m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +void m00150s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + salt_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00150_a3-pure.cl b/OpenCL/m00150_a3-pure.cl deleted file mode 100644 index 27c7dc7dd..000000000 --- a/OpenCL/m00150_a3-pure.cl +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_hmac_ctx_vector_t ctx; - - sha1_hmac_init_vector (&ctx, w, pw_len); - - sha1_hmac_update_vector (&ctx, s, salt_len); - - sha1_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_hmac_ctx_vector_t ctx; - - sha1_hmac_init_vector (&ctx, w, pw_len); - - sha1_hmac_update_vector (&ctx, s, salt_len); - - sha1_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00150_a3.cl b/OpenCL/m00150_a3.cl index ec392c959..27c7dc7dd 100644 --- a/OpenCL/m00150_a3.cl +++ b/OpenCL/m00150_a3.cl @@ -11,365 +11,90 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -void m00150m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00150_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha1_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha1_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_update_vector (&ctx, s, salt_len); - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha1_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + salt_len) * 8; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[5]; - - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00150s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00150_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + if (gid >= gid_max) return; /** * digest @@ -383,352 +108,63 @@ void m00150s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha1_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha1_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[5]; - u32x opad[5]; + sha1_hmac_update_vector (&ctx, s, salt_len); - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha1_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + salt_len) * 8; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[5]; - - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00150_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00150_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00150_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00150_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00150_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00150_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00150s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00160_a0-optimized.cl b/OpenCL/m00160_a0-optimized.cl new file mode 100644 index 000000000..d117232a3 --- /dev/null +++ b/OpenCL/m00160_a0-optimized.cl @@ -0,0 +1,518 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (64 + out_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (64 + out_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00160_a0-pure.cl b/OpenCL/m00160_a0-pure.cl deleted file mode 100644 index c87a1bab7..000000000 --- a/OpenCL/m00160_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_t ctx0; - - sha1_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_hmac_ctx_t ctx = ctx0; - - sha1_hmac_update_swap (&ctx, w, pw_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_t ctx0; - - sha1_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_hmac_ctx_t ctx = ctx0; - - sha1_hmac_update_swap (&ctx, w, pw_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00160_a0.cl b/OpenCL/m00160_a0.cl index d117232a3..c87a1bab7 100644 --- a/OpenCL/m00160_a0.cl +++ b/OpenCL/m00160_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,451 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[5]; - u32x opad[5]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha1_hmac_ctx_t ctx0; + + sha1_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (64 + out_len) * 8; + sha1_hmac_final (&ctx); - u32x digest[5]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[5]; - u32x opad[5]; - - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -469,50 +106,59 @@ __kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_hmac_ctx_t ctx0; + + sha1_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (64 + out_len) * 8; + sha1_hmac_final (&ctx); - u32x digest[5]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00160_a1-optimized.cl b/OpenCL/m00160_a1-optimized.cl new file mode 100644 index 000000000..c325dbcab --- /dev/null +++ b/OpenCL/m00160_a1-optimized.cl @@ -0,0 +1,636 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (64 + pw_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (64 + pw_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00160_a1-pure.cl b/OpenCL/m00160_a1-pure.cl deleted file mode 100644 index 6a4907c55..000000000 --- a/OpenCL/m00160_a1-pure.cl +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_t ctx0; - - sha1_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha1_hmac_ctx_t ctx = ctx0; - - sha1_hmac_update (&ctx, c, pw_len + comb_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_t ctx0; - - sha1_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha1_hmac_ctx_t ctx = ctx0; - - sha1_hmac_update (&ctx, c, pw_len + comb_len); - - sha1_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00160_a1.cl b/OpenCL/m00160_a1.cl index c325dbcab..6a4907c55 100644 --- a/OpenCL/m00160_a1.cl +++ b/OpenCL/m00160_a1.cl @@ -3,518 +3,115 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[5]; - u32x opad[5]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha1_hmac_ctx_t ctx0; + + sha1_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha1_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (64 + pw_len) * 8; + sha1_hmac_update (&ctx, c, pw_len + comb_len); - u32x digest[5]; + sha1_hmac_final (&ctx); - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[5]; - u32x opad[5]; - - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -527,110 +124,79 @@ __kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_hmac_ctx_t ctx0; + + sha1_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha1_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (64 + pw_len) * 8; + sha1_hmac_update (&ctx, c, pw_len + comb_len); - u32x digest[5]; + sha1_hmac_final (&ctx); - hmac_sha1_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00160_a3-optimized.cl b/OpenCL/m00160_a3-optimized.cl new file mode 100644 index 000000000..951c73a71 --- /dev/null +++ b/OpenCL/m00160_a3-optimized.cl @@ -0,0 +1,734 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) +{ + u32x A = digest[0]; + u32x B = digest[1]; + u32x C = digest[2]; + u32x D = digest[3]; + u32x E = digest[4]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); + SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); + SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); + SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); + SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); + SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); + + digest[0] += A; + digest[1] += B; + digest[2] += C; + digest[3] += D; + digest[4] += E; +} + +void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA1M_A; + ipad[1] = SHA1M_B; + ipad[2] = SHA1M_C; + ipad[3] = SHA1M_D; + ipad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA1M_A; + opad[1] = SHA1M_B; + opad[2] = SHA1M_C; + opad[3] = SHA1M_D; + opad[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + + sha1_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = 0x80000000; + w1[2] = 0; + w1[3] = 0; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 20) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + + sha1_transform (w0, w1, w2, w3, digest); +} + +void m00160m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + pw_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +void m00160s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[5]; + u32x opad[5]; + + hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + pw_len) * 8; + + u32x digest[5]; + + hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + } +} + +__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00160_a3-pure.cl b/OpenCL/m00160_a3-pure.cl deleted file mode 100644 index cdcca3d64..000000000 --- a/OpenCL/m00160_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_vector_t ctx0; - - sha1_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_hmac_ctx_vector_t ctx = ctx0; - - sha1_hmac_update_vector (&ctx, w, pw_len); - - sha1_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha1_hmac_ctx_vector_t ctx0; - - sha1_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_hmac_ctx_vector_t ctx = ctx0; - - sha1_hmac_update_vector (&ctx, w, pw_len); - - sha1_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00160_a3.cl b/OpenCL/m00160_a3.cl index 951c73a71..cdcca3d64 100644 --- a/OpenCL/m00160_a3.cl +++ b/OpenCL/m00160_a3.cl @@ -11,396 +11,92 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void sha1_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[5]) -{ - u32x A = digest[0]; - u32x B = digest[1]; - u32x C = digest[2]; - u32x D = digest[3]; - u32x E = digest[4]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w0_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w1_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w2_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w3_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w4_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, w5_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, w6_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, w7_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, w8_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, w9_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wa_t); - SHA1_STEP (SHA1_F0o, E, A, B, C, D, wb_t); - SHA1_STEP (SHA1_F0o, D, E, A, B, C, wc_t); - SHA1_STEP (SHA1_F0o, C, D, E, A, B, wd_t); - SHA1_STEP (SHA1_F0o, B, C, D, E, A, we_t); - SHA1_STEP (SHA1_F0o, A, B, C, D, E, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, E, A, B, C, D, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, D, E, A, B, C, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, C, D, E, A, B, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, B, C, D, E, A, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, A, B, C, D, E, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, E, A, B, C, D, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, D, E, A, B, C, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, C, D, E, A, B, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, B, C, D, E, A, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, A, B, C, D, E, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, E, A, B, C, D, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, D, E, A, B, C, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, C, D, E, A, B, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, B, C, D, E, A, wf_t); - - digest[0] += A; - digest[1] += B; - digest[2] += C; - digest[3] += D; - digest[4] += E; -} - -void hmac_sha1_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA1M_A; - ipad[1] = SHA1M_B; - ipad[2] = SHA1M_C; - ipad[3] = SHA1M_D; - ipad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA1M_A; - opad[1] = SHA1M_B; - opad[2] = SHA1M_C; - opad[3] = SHA1M_D; - opad[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha1_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[5], u32x opad[5], u32x digest[5]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - - sha1_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = 0x80000000; - w1[2] = 0; - w1[3] = 0; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 20) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - - sha1_transform (w0, w1, w2, w3, digest); -} - -void m00160m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00160_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32x s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[5]; - u32x opad[5]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha1_hmac_ctx_vector_t ctx0; + + sha1_hmac_init_vector (&ctx0, s, salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + pw_len) * 8; + w[0] = w0; - u32x digest[5]; + sha1_hmac_ctx_vector_t ctx = ctx0; - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha1_hmac_update_vector (&ctx, w, pw_len); - COMPARE_M_SIMD (digest[3], digest[4], digest[2], digest[1]); + sha1_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00160s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00160_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[5]; - u32x opad[5]; - - hmac_sha1_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + if (gid >= gid_max) return; /** * digest @@ -414,321 +110,65 @@ void m00160s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha1_hmac_ctx_vector_t ctx0; + + sha1_hmac_init_vector (&ctx0, s, salt_len); + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + pw_len) * 8; + w[0] = w0; - u32x digest[5]; + sha1_hmac_ctx_vector_t ctx = ctx0; - hmac_sha1_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha1_hmac_update_vector (&ctx, w, pw_len); - COMPARE_S_SIMD (digest[3], digest[4], digest[2], digest[1]); + sha1_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00160_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00160_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00160_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00160_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00160_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00160_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00160s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00300_a0-optimized.cl b/OpenCL/m00300_a0-optimized.cl new file mode 100644 index 000000000..71b87e689 --- /dev/null +++ b/OpenCL/m00300_a0-optimized.cl @@ -0,0 +1,646 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + w0_t = a; + w1_t = b; + w2_t = c; + w3_t = d; + w4_t = e; + w5_t = 0x80000000; + w6_t = 0; + w7_t = 0; + w8_t = 0; + w9_t = 0; + wa_t = 0; + wb_t = 0; + wc_t = 0; + wd_t = 0; + we_t = 0; + wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + w0_t = a; + w1_t = b; + w2_t = c; + w3_t = d; + w4_t = e; + w5_t = 0x80000000; + w6_t = 0; + w7_t = 0; + w8_t = 0; + w9_t = 0; + wa_t = 0; + wb_t = 0; + wc_t = 0; + wd_t = 0; + we_t = 0; + wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00300_a0-pure.cl b/OpenCL/m00300_a0-pure.cl deleted file mode 100644 index 04e7f4940..000000000 --- a/OpenCL/m00300_a0-pure.cl +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_swap (&ctx0, w, pw_len); - - sha1_final (&ctx0); - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - ctx.w1[0] = ctx0.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_swap (&ctx0, w, pw_len); - - sha1_final (&ctx0); - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - ctx.w1[0] = ctx0.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00300_a0.cl b/OpenCL/m00300_a0.cl index 71b87e689..04e7f4940 100644 --- a/OpenCL/m00300_a0.cl +++ b/OpenCL/m00300_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,343 +12,96 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_init (&ctx0); - /** - * sha1 - */ + sha1_update_swap (&ctx0, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha1_final (&ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_ctx_t ctx; - #undef K - #define K SHA1C00 + sha1_init (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; + ctx.w1[0] = ctx0.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - #undef K - #define K SHA1C01 + sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - w0_t = a; - w1_t = b; - w2_t = c; - w3_t = d; - w4_t = e; - w5_t = 0x80000000; - w6_t = 0; - w7_t = 0; - w8_t = 0; - w9_t = 0; - wa_t = 0; - wb_t = 0; - wc_t = 0; - wd_t = 0; - we_t = 0; - wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -362,285 +115,66 @@ __kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha1_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha1_init (&ctx0); - /** - * sha1 - */ + sha1_update_swap (&ctx0, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha1_final (&ctx0); - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_ctx_t ctx; - #undef K - #define K SHA1C00 + sha1_init (&ctx); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; + ctx.w1[0] = ctx0.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - #undef K - #define K SHA1C01 + sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - w0_t = a; - w1_t = b; - w2_t = c; - w3_t = d; - w4_t = e; - w5_t = 0x80000000; - w6_t = 0; - w7_t = 0; - w8_t = 0; - w9_t = 0; - wa_t = 0; - wb_t = 0; - wc_t = 0; - wd_t = 0; - we_t = 0; - wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00300_a1-optimized.cl b/OpenCL/m00300_a1-optimized.cl new file mode 100644 index 000000000..93afec569 --- /dev/null +++ b/OpenCL/m00300_a1-optimized.cl @@ -0,0 +1,760 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + w0_t = a; + w1_t = b; + w2_t = c; + w3_t = d; + w4_t = e; + w5_t = 0x80000000; + w6_t = 0; + w7_t = 0; + w8_t = 0; + w9_t = 0; + wa_t = 0; + wb_t = 0; + wc_t = 0; + wd_t = 0; + we_t = 0; + wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha1 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + w0_t = a; + w1_t = b; + w2_t = c; + w3_t = d; + w4_t = e; + w5_t = 0x80000000; + w6_t = 0; + w7_t = 0; + w8_t = 0; + w9_t = 0; + wa_t = 0; + wb_t = 0; + wc_t = 0; + wd_t = 0; + we_t = 0; + wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00300_a1-pure.cl b/OpenCL/m00300_a1-pure.cl deleted file mode 100644 index b694960bc..000000000 --- a/OpenCL/m00300_a1-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx1 = ctx0; - - sha1_update_global_swap (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx1); - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - ctx.w0[0] = ctx1.h[0]; - ctx.w0[1] = ctx1.h[1]; - ctx.w0[2] = ctx1.h[2]; - ctx.w0[3] = ctx1.h[3]; - ctx.w1[0] = ctx1.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha1_ctx_t ctx0; - - sha1_init (&ctx0); - - sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha1_ctx_t ctx1 = ctx0; - - sha1_update_global_swap (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha1_final (&ctx1); - - sha1_ctx_t ctx; - - sha1_init (&ctx); - - ctx.w0[0] = ctx1.h[0]; - ctx.w0[1] = ctx1.h[1]; - ctx.w0[2] = ctx1.h[2]; - ctx.w0[3] = ctx1.h[3]; - ctx.w1[0] = ctx1.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00300_a1.cl b/OpenCL/m00300_a1.cl index 93afec569..b694960bc 100644 --- a/OpenCL/m00300_a1.cl +++ b/OpenCL/m00300_a1.cl @@ -3,408 +3,92 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha1.cl" -__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha1_ctx_t ctx0; - if (gid >= gid_max) return; + sha1_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_ctx_t ctx; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + sha1_init (&ctx); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + ctx.w0[0] = ctx1.h[0]; + ctx.w0[1] = ctx1.h[1]; + ctx.w0[2] = ctx1.h[2]; + ctx.w0[3] = ctx1.h[3]; + ctx.w1[0] = ctx1.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - w0_t = a; - w1_t = b; - w2_t = c; - w3_t = d; - w4_t = e; - w5_t = 0x80000000; - w6_t = 0; - w7_t = 0; - w8_t = 0; - w9_t = 0; - wa_t = 0; - wb_t = 0; - wc_t = 0; - wd_t = 0; - we_t = 0; - wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -418,343 +102,55 @@ __kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + sha1_ctx_t ctx0; + + sha1_init (&ctx0); + + sha1_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha1_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha1_update_global_swap (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha1_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha1_ctx_t ctx; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + sha1_init (&ctx); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + ctx.w0[0] = ctx1.h[0]; + ctx.w0[1] = ctx1.h[1]; + ctx.w0[2] = ctx1.h[2]; + ctx.w0[3] = ctx1.h[3]; + ctx.w1[0] = ctx1.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + sha1_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha1 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - w0_t = a; - w1_t = b; - w2_t = c; - w3_t = d; - w4_t = e; - w5_t = 0x80000000; - w6_t = 0; - w7_t = 0; - w8_t = 0; - w9_t = 0; - wa_t = 0; - wb_t = 0; - wc_t = 0; - wd_t = 0; - we_t = 0; - wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00300_a3-optimized.cl b/OpenCL/m00300_a3-optimized.cl new file mode 100644 index 000000000..84c142009 --- /dev/null +++ b/OpenCL/m00300_a3-optimized.cl @@ -0,0 +1,1000 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m00300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + u32x w0_t = a; + u32x w1_t = b; + u32x w2_t = c; + u32x w3_t = d; + u32x w4_t = e; + u32x w5_t = 0x80000000; + u32x w6_t = 0; + u32x w7_t = 0; + u32x w8_t = 0; + u32x w9_t = 0; + u32x wa_t = 0; + u32x wb_t = 0; + u32x wc_t = 0; + u32x wd_t = 0; + u32x we_t = 0; + u32x wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_M_SIMD (d, e, c, b); + } +} + +void m00300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); + const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); + const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); + const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); + const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); + const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); + const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); + const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); + const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); + const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); + const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); + const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); + const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); + const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); + const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); + const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); + const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); + const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); + const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); + const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); + const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); + const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); + const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); + const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); + const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); + const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); + const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); + const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); + const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); + const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); + const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); + const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); + const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); + const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); + const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); + const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); + const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); + const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); + const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); + const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); + const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); + const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); + const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); + const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); + const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); + const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); + const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); + const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); + const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); + const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); + const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); + const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); + const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); + const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); + const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); + const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); + const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); + const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); + const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); + const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + + const u32 c_17sK = c_17s + SHA1C00; + const u32 c_18sK = c_18s + SHA1C00; + const u32 c_20sK = c_20s + SHA1C01; + const u32 c_21sK = c_21s + SHA1C01; + const u32 c_23sK = c_23s + SHA1C01; + const u32 c_26sK = c_26s + SHA1C01; + const u32 c_27sK = c_27s + SHA1C01; + const u32 c_29sK = c_29s + SHA1C01; + const u32 c_33sK = c_33s + SHA1C01; + const u32 c_39sK = c_39s + SHA1C01; + const u32 c_41sK = c_41s + SHA1C02; + const u32 c_45sK = c_45s + SHA1C02; + const u32 c_53sK = c_53s + SHA1C02; + const u32 c_65sK = c_65s + SHA1C03; + const u32 c_69sK = c_69s + SHA1C03; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + const u32 e_rev = rotl32_S (search[1], 2u); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + const u32x w0s01 = rotl32 (w0, 1u); + const u32x w0s02 = rotl32 (w0, 2u); + const u32x w0s03 = rotl32 (w0, 3u); + const u32x w0s04 = rotl32 (w0, 4u); + const u32x w0s05 = rotl32 (w0, 5u); + const u32x w0s06 = rotl32 (w0, 6u); + const u32x w0s07 = rotl32 (w0, 7u); + const u32x w0s08 = rotl32 (w0, 8u); + const u32x w0s09 = rotl32 (w0, 9u); + const u32x w0s10 = rotl32 (w0, 10u); + const u32x w0s11 = rotl32 (w0, 11u); + const u32x w0s12 = rotl32 (w0, 12u); + const u32x w0s13 = rotl32 (w0, 13u); + const u32x w0s14 = rotl32 (w0, 14u); + const u32x w0s15 = rotl32 (w0, 15u); + const u32x w0s16 = rotl32 (w0, 16u); + const u32x w0s17 = rotl32 (w0, 17u); + const u32x w0s18 = rotl32 (w0, 18u); + const u32x w0s19 = rotl32 (w0, 19u); + const u32x w0s20 = rotl32 (w0, 20u); + + const u32x w0s04___w0s06 = w0s04 ^ w0s06; + const u32x w0s04___w0s08 = w0s04 ^ w0s08; + const u32x w0s08___w0s12 = w0s08 ^ w0s12; + const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + + u32x a = SHA1M_A; + u32x b = SHA1M_B; + u32x c = SHA1M_C; + u32x d = SHA1M_D; + u32x e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + + SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); + SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); + SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + + #undef K + #define K SHA1C01 + + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); + SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); + SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + + #undef K + #define K SHA1C02 + + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); + SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); + SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); + SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); + SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); + SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); + SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); + SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); + SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + + #undef K + #define K SHA1C03 + + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); + SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); + SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); + SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); + + const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); + const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); + const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); + const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); + + const u32x w0s21 = rotl32 (w0, 21u); + const u32x w0s22 = rotl32 (w0, 22U); + + SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); + SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); + SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); + SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); + + a += SHA1M_A; + b += SHA1M_B; + c += SHA1M_C; + d += SHA1M_D; + e += SHA1M_E; + + u32x w0_t = a; + u32x w1_t = b; + u32x w2_t = c; + u32x w3_t = d; + u32x w4_t = e; + u32x w5_t = 0x80000000; + u32x w6_t = 0; + u32x w7_t = 0; + u32x w8_t = 0; + u32x w9_t = 0; + u32x wa_t = 0; + u32x wb_t = 0; + u32x wc_t = 0; + u32x wd_t = 0; + u32x we_t = 0; + u32x wf_t = 20 * 8; + + a = SHA1M_A; + b = SHA1M_B; + c = SHA1M_C; + d = SHA1M_D; + e = SHA1M_E; + + #undef K + #define K SHA1C00 + + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); + SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); + SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); + SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); + SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); + SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); + + #undef K + #define K SHA1C01 + + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); + + #undef K + #define K SHA1C02 + + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); + + #undef K + #define K SHA1C03 + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); + w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); + w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); + w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); + w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); + w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); + w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); + w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); + w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); + w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); + w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); + wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); + wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); + + if (MATCHES_NONE_VS (e, e_rev)) continue; + + wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); + wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); + we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); + wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); + + COMPARE_S_SIMD (d, e, c, b); + } +} + +__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00300_a3-pure.cl b/OpenCL/m00300_a3-pure.cl deleted file mode 100644 index 6a9763f3f..000000000 --- a/OpenCL/m00300_a3-pure.cl +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha1.cl" - -__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx0; - - sha1_init_vector (&ctx0); - - sha1_update_vector (&ctx0, w, pw_len); - - sha1_final_vector (&ctx0); - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - ctx.w1[0] = ctx0.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha1_ctx_vector_t ctx0; - - sha1_init_vector (&ctx0); - - sha1_update_vector (&ctx0, w, pw_len); - - sha1_final_vector (&ctx0); - - sha1_ctx_vector_t ctx; - - sha1_init_vector (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - ctx.w1[0] = ctx0.h[4]; - ctx.w1[1] = 0x80000000; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 0; - ctx.w3[3] = 20 * 8; - - sha1_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00300_a3.cl b/OpenCL/m00300_a3.cl index 84c142009..6a9763f3f 100644 --- a/OpenCL/m00300_a3.cl +++ b/OpenCL/m00300_a3.cl @@ -11,102 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha1.cl" -void m00300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); + const u32 pw_len = pws[gid].pw_len; - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -114,362 +53,58 @@ void m00300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx0; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx0); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx0, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_final_vector (&ctx0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_ctx_vector_t ctx; - #undef K - #define K SHA1C01 + sha1_init_vector (&ctx); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; + ctx.w1[0] = ctx0.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - #undef K - #define K SHA1C02 + sha1_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - u32x w0_t = a; - u32x w1_t = b; - u32x w2_t = c; - u32x w3_t = d; - u32x w4_t = e; - u32x w5_t = 0x80000000; - u32x w6_t = 0; - u32x w7_t = 0; - u32x w8_t = 0; - u32x w9_t = 0; - u32x wa_t = 0; - u32x wb_t = 0; - u32x wc_t = 0; - u32x wd_t = 0; - u32x we_t = 0; - u32x wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_M_SIMD (d, e, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 c_16s = rotl32_S ((w[13] ^ w[ 8] ^ w[ 2] ), 1u); - const u32 c_17s = rotl32_S ((w[14] ^ w[ 9] ^ w[ 3] ^ w[ 1]), 1u); - const u32 c_18s = rotl32_S ((w[15] ^ w[10] ^ w[ 4] ^ w[ 2]), 1u); - const u32 c_19s = rotl32_S ((c_16s ^ w[11] ^ w[ 5] ^ w[ 3]), 1u); - const u32 c_20s = rotl32_S ((c_17s ^ w[12] ^ w[ 6] ^ w[ 4]), 1u); - const u32 c_21s = rotl32_S ((c_18s ^ w[13] ^ w[ 7] ^ w[ 5]), 1u); - const u32 c_22s = rotl32_S ((c_19s ^ w[14] ^ w[ 8] ^ w[ 6]), 1u); - const u32 c_23s = rotl32_S ((c_20s ^ w[15] ^ w[ 9] ^ w[ 7]), 1u); - const u32 c_24s = rotl32_S ((c_21s ^ c_16s ^ w[10] ^ w[ 8]), 1u); - const u32 c_25s = rotl32_S ((c_22s ^ c_17s ^ w[11] ^ w[ 9]), 1u); - const u32 c_26s = rotl32_S ((c_23s ^ c_18s ^ w[12] ^ w[10]), 1u); - const u32 c_27s = rotl32_S ((c_24s ^ c_19s ^ w[13] ^ w[11]), 1u); - const u32 c_28s = rotl32_S ((c_25s ^ c_20s ^ w[14] ^ w[12]), 1u); - const u32 c_29s = rotl32_S ((c_26s ^ c_21s ^ w[15] ^ w[13]), 1u); - const u32 c_30s = rotl32_S ((c_27s ^ c_22s ^ c_16s ^ w[14]), 1u); - const u32 c_31s = rotl32_S ((c_28s ^ c_23s ^ c_17s ^ w[15]), 1u); - const u32 c_32s = rotl32_S ((c_29s ^ c_24s ^ c_18s ^ c_16s), 1u); - const u32 c_33s = rotl32_S ((c_30s ^ c_25s ^ c_19s ^ c_17s), 1u); - const u32 c_34s = rotl32_S ((c_31s ^ c_26s ^ c_20s ^ c_18s), 1u); - const u32 c_35s = rotl32_S ((c_32s ^ c_27s ^ c_21s ^ c_19s), 1u); - const u32 c_36s = rotl32_S ((c_33s ^ c_28s ^ c_22s ^ c_20s), 1u); - const u32 c_37s = rotl32_S ((c_34s ^ c_29s ^ c_23s ^ c_21s), 1u); - const u32 c_38s = rotl32_S ((c_35s ^ c_30s ^ c_24s ^ c_22s), 1u); - const u32 c_39s = rotl32_S ((c_36s ^ c_31s ^ c_25s ^ c_23s), 1u); - const u32 c_40s = rotl32_S ((c_37s ^ c_32s ^ c_26s ^ c_24s), 1u); - const u32 c_41s = rotl32_S ((c_38s ^ c_33s ^ c_27s ^ c_25s), 1u); - const u32 c_42s = rotl32_S ((c_39s ^ c_34s ^ c_28s ^ c_26s), 1u); - const u32 c_43s = rotl32_S ((c_40s ^ c_35s ^ c_29s ^ c_27s), 1u); - const u32 c_44s = rotl32_S ((c_41s ^ c_36s ^ c_30s ^ c_28s), 1u); - const u32 c_45s = rotl32_S ((c_42s ^ c_37s ^ c_31s ^ c_29s), 1u); - const u32 c_46s = rotl32_S ((c_43s ^ c_38s ^ c_32s ^ c_30s), 1u); - const u32 c_47s = rotl32_S ((c_44s ^ c_39s ^ c_33s ^ c_31s), 1u); - const u32 c_48s = rotl32_S ((c_45s ^ c_40s ^ c_34s ^ c_32s), 1u); - const u32 c_49s = rotl32_S ((c_46s ^ c_41s ^ c_35s ^ c_33s), 1u); - const u32 c_50s = rotl32_S ((c_47s ^ c_42s ^ c_36s ^ c_34s), 1u); - const u32 c_51s = rotl32_S ((c_48s ^ c_43s ^ c_37s ^ c_35s), 1u); - const u32 c_52s = rotl32_S ((c_49s ^ c_44s ^ c_38s ^ c_36s), 1u); - const u32 c_53s = rotl32_S ((c_50s ^ c_45s ^ c_39s ^ c_37s), 1u); - const u32 c_54s = rotl32_S ((c_51s ^ c_46s ^ c_40s ^ c_38s), 1u); - const u32 c_55s = rotl32_S ((c_52s ^ c_47s ^ c_41s ^ c_39s), 1u); - const u32 c_56s = rotl32_S ((c_53s ^ c_48s ^ c_42s ^ c_40s), 1u); - const u32 c_57s = rotl32_S ((c_54s ^ c_49s ^ c_43s ^ c_41s), 1u); - const u32 c_58s = rotl32_S ((c_55s ^ c_50s ^ c_44s ^ c_42s), 1u); - const u32 c_59s = rotl32_S ((c_56s ^ c_51s ^ c_45s ^ c_43s), 1u); - const u32 c_60s = rotl32_S ((c_57s ^ c_52s ^ c_46s ^ c_44s), 1u); - const u32 c_61s = rotl32_S ((c_58s ^ c_53s ^ c_47s ^ c_45s), 1u); - const u32 c_62s = rotl32_S ((c_59s ^ c_54s ^ c_48s ^ c_46s), 1u); - const u32 c_63s = rotl32_S ((c_60s ^ c_55s ^ c_49s ^ c_47s), 1u); - const u32 c_64s = rotl32_S ((c_61s ^ c_56s ^ c_50s ^ c_48s), 1u); - const u32 c_65s = rotl32_S ((c_62s ^ c_57s ^ c_51s ^ c_49s), 1u); - const u32 c_66s = rotl32_S ((c_63s ^ c_58s ^ c_52s ^ c_50s), 1u); - const u32 c_67s = rotl32_S ((c_64s ^ c_59s ^ c_53s ^ c_51s), 1u); - const u32 c_68s = rotl32_S ((c_65s ^ c_60s ^ c_54s ^ c_52s), 1u); - const u32 c_69s = rotl32_S ((c_66s ^ c_61s ^ c_55s ^ c_53s), 1u); - const u32 c_70s = rotl32_S ((c_67s ^ c_62s ^ c_56s ^ c_54s), 1u); - const u32 c_71s = rotl32_S ((c_68s ^ c_63s ^ c_57s ^ c_55s), 1u); - const u32 c_72s = rotl32_S ((c_69s ^ c_64s ^ c_58s ^ c_56s), 1u); - const u32 c_73s = rotl32_S ((c_70s ^ c_65s ^ c_59s ^ c_57s), 1u); - const u32 c_74s = rotl32_S ((c_71s ^ c_66s ^ c_60s ^ c_58s), 1u); - const u32 c_75s = rotl32_S ((c_72s ^ c_67s ^ c_61s ^ c_59s), 1u); - - const u32 c_17sK = c_17s + SHA1C00; - const u32 c_18sK = c_18s + SHA1C00; - const u32 c_20sK = c_20s + SHA1C01; - const u32 c_21sK = c_21s + SHA1C01; - const u32 c_23sK = c_23s + SHA1C01; - const u32 c_26sK = c_26s + SHA1C01; - const u32 c_27sK = c_27s + SHA1C01; - const u32 c_29sK = c_29s + SHA1C01; - const u32 c_33sK = c_33s + SHA1C01; - const u32 c_39sK = c_39s + SHA1C01; - const u32 c_41sK = c_41s + SHA1C02; - const u32 c_45sK = c_45s + SHA1C02; - const u32 c_53sK = c_53s + SHA1C02; - const u32 c_65sK = c_65s + SHA1C03; - const u32 c_69sK = c_69s + SHA1C03; + if (gid >= gid_max) return; /** * digest @@ -484,16 +119,27 @@ void m00300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - const u32 e_rev = rotl32_S (search[1], 2u); + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -501,500 +147,44 @@ void m00300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - const u32x w0s01 = rotl32 (w0, 1u); - const u32x w0s02 = rotl32 (w0, 2u); - const u32x w0s03 = rotl32 (w0, 3u); - const u32x w0s04 = rotl32 (w0, 4u); - const u32x w0s05 = rotl32 (w0, 5u); - const u32x w0s06 = rotl32 (w0, 6u); - const u32x w0s07 = rotl32 (w0, 7u); - const u32x w0s08 = rotl32 (w0, 8u); - const u32x w0s09 = rotl32 (w0, 9u); - const u32x w0s10 = rotl32 (w0, 10u); - const u32x w0s11 = rotl32 (w0, 11u); - const u32x w0s12 = rotl32 (w0, 12u); - const u32x w0s13 = rotl32 (w0, 13u); - const u32x w0s14 = rotl32 (w0, 14u); - const u32x w0s15 = rotl32 (w0, 15u); - const u32x w0s16 = rotl32 (w0, 16u); - const u32x w0s17 = rotl32 (w0, 17u); - const u32x w0s18 = rotl32 (w0, 18u); - const u32x w0s19 = rotl32 (w0, 19u); - const u32x w0s20 = rotl32 (w0, 20u); + w[0] = w0; - const u32x w0s04___w0s06 = w0s04 ^ w0s06; - const u32x w0s04___w0s08 = w0s04 ^ w0s08; - const u32x w0s08___w0s12 = w0s08 ^ w0s12; - const u32x w0s04___w0s06___w0s07 = w0s04___w0s06 ^ w0s07; + sha1_ctx_vector_t ctx0; - u32x a = SHA1M_A; - u32x b = SHA1M_B; - u32x c = SHA1M_C; - u32x d = SHA1M_D; - u32x e = SHA1M_E; + sha1_init_vector (&ctx0); - #undef K - #define K SHA1C00 + sha1_update_vector (&ctx0, w, pw_len); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 1]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 2]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 3]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 4]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[ 5]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[ 6]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[ 7]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[ 8]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[ 9]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[10]); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w[11]); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w[12]); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w[13]); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w[14]); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w[15]); + sha1_final_vector (&ctx0); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, (c_16s ^ w0s01)); - SHA1_STEPX(SHA1_F0o, d, e, a, b, c, (c_17sK)); - SHA1_STEPX(SHA1_F0o, c, d, e, a, b, (c_18sK)); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, (c_19s ^ w0s02)); + sha1_ctx_vector_t ctx; - #undef K - #define K SHA1C01 + sha1_init_vector (&ctx); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_20sK)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_21sK)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_22s ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_23sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_24s ^ w0s02)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_25s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , e, a, b, c, d, (c_26sK)); - SHA1_STEPX(SHA1_F1 , d, e, a, b, c, (c_27sK)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_28s ^ w0s05)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_29sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_30s ^ w0s02 ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_31s ^ w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_32s ^ w0s02 ^ w0s03)); - SHA1_STEPX(SHA1_F1 , c, d, e, a, b, (c_33sK)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_34s ^ w0s07)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_35s ^ w0s04)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_36s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_37s ^ w0s08)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_38s ^ w0s04)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_39sK)); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; + ctx.w1[0] = ctx0.h[4]; + ctx.w1[1] = 0x80000000; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 0; + ctx.w3[3] = 20 * 8; - #undef K - #define K SHA1C02 + sha1_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_40s ^ w0s04 ^ w0s09)); - SHA1_STEPX(SHA1_F2o, e, a, b, c, d, (c_41sK)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_42s ^ w0s06 ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_43s ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_44s ^ w0s03 ^ w0s06 ^ w0s07)); - SHA1_STEPX(SHA1_F2o, a, b, c, d, e, (c_45sK)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_46s ^ w0s04 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_47s ^ w0s04___w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_48s ^ w0s03 ^ w0s04___w0s08 ^ w0s05 ^ w0s10)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_49s ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_50s ^ w0s08)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_51s ^ w0s04___w0s06)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_52s ^ w0s04___w0s08 ^ w0s13)); - SHA1_STEPX(SHA1_F2o, c, d, e, a, b, (c_53sK)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_54s ^ w0s07 ^ w0s10 ^ w0s12)); - SHA1_STEP (SHA1_F2o, a, b, c, d, e, (c_55s ^ w0s14)); - SHA1_STEP (SHA1_F2o, e, a, b, c, d, (c_56s ^ w0s04___w0s06___w0s07 ^ w0s10 ^ w0s11)); - SHA1_STEP (SHA1_F2o, d, e, a, b, c, (c_57s ^ w0s08)); - SHA1_STEP (SHA1_F2o, c, d, e, a, b, (c_58s ^ w0s04___w0s08 ^ w0s15)); - SHA1_STEP (SHA1_F2o, b, c, d, e, a, (c_59s ^ w0s08___w0s12)); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - #undef K - #define K SHA1C03 - - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_60s ^ w0s04 ^ w0s08___w0s12 ^ w0s07 ^ w0s14)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_61s ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_62s ^ w0s04___w0s06 ^ w0s08___w0s12)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_63s ^ w0s08)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_64s ^ w0s04___w0s06___w0s07 ^ w0s08___w0s12 ^ w0s17)); - SHA1_STEPX(SHA1_F1 , a, b, c, d, e, (c_65sK)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_66s ^ w0s14 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_67s ^ w0s08 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_68s ^ w0s11 ^ w0s14 ^ w0s15)); - SHA1_STEPX(SHA1_F1 , b, c, d, e, a, (c_69sK)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_70s ^ w0s12 ^ w0s19)); - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_71s ^ w0s12 ^ w0s16)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_72s ^ w0s05 ^ w0s11 ^ w0s12 ^ w0s13 ^ w0s16 ^ w0s18)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_73s ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_74s ^ w0s08 ^ w0s16)); - SHA1_STEP (SHA1_F1 , a, b, c, d, e, (c_75s ^ w0s06 ^ w0s12 ^ w0s14)); - - const u32x c_76s = rotl32 ((c_73s ^ c_68s ^ c_62s ^ c_60s), 1u); - const u32x c_77s = rotl32 ((c_74s ^ c_69s ^ c_63s ^ c_61s), 1u); - const u32x c_78s = rotl32 ((c_75s ^ c_70s ^ c_64s ^ c_62s), 1u); - const u32x c_79s = rotl32 ((c_76s ^ c_71s ^ c_65s ^ c_63s), 1u); - - const u32x w0s21 = rotl32 (w0, 21u); - const u32x w0s22 = rotl32 (w0, 22U); - - SHA1_STEP (SHA1_F1 , e, a, b, c, d, (c_76s ^ w0s07 ^ w0s08___w0s12 ^ w0s16 ^ w0s21)); - SHA1_STEP (SHA1_F1 , d, e, a, b, c, (c_77s)); - SHA1_STEP (SHA1_F1 , c, d, e, a, b, (c_78s ^ w0s07 ^ w0s08 ^ w0s15 ^ w0s18 ^ w0s20)); - SHA1_STEP (SHA1_F1 , b, c, d, e, a, (c_79s ^ w0s08 ^ w0s22)); - - a += SHA1M_A; - b += SHA1M_B; - c += SHA1M_C; - d += SHA1M_D; - e += SHA1M_E; - - u32x w0_t = a; - u32x w1_t = b; - u32x w2_t = c; - u32x w3_t = d; - u32x w4_t = e; - u32x w5_t = 0x80000000; - u32x w6_t = 0; - u32x w7_t = 0; - u32x w8_t = 0; - u32x w9_t = 0; - u32x wa_t = 0; - u32x wb_t = 0; - u32x wc_t = 0; - u32x wd_t = 0; - u32x we_t = 0; - u32x wf_t = 20 * 8; - - a = SHA1M_A; - b = SHA1M_B; - c = SHA1M_C; - d = SHA1M_D; - e = SHA1M_E; - - #undef K - #define K SHA1C00 - - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w0_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w1_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w2_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w3_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w4_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, w5_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, w6_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, w7_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, w8_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, w9_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wa_t); - SHA1_STEP (SHA1_F0o, e, a, b, c, d, wb_t); - SHA1_STEP (SHA1_F0o, d, e, a, b, c, wc_t); - SHA1_STEP (SHA1_F0o, c, d, e, a, b, wd_t); - SHA1_STEP (SHA1_F0o, b, c, d, e, a, we_t); - SHA1_STEP (SHA1_F0o, a, b, c, d, e, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F0o, e, a, b, c, d, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F0o, d, e, a, b, c, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F0o, c, d, e, a, b, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F0o, b, c, d, e, a, w3_t); - - #undef K - #define K SHA1C01 - - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w7_t); - - #undef K - #define K SHA1C02 - - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wb_t); - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F2o, a, b, c, d, e, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F2o, e, a, b, c, d, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F2o, d, e, a, b, c, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F2o, c, d, e, a, b, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F2o, b, c, d, e, a, wb_t); - - #undef K - #define K SHA1C03 - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, wf_t); - w0_t = rotl32 ((wd_t ^ w8_t ^ w2_t ^ w0_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w0_t); - w1_t = rotl32 ((we_t ^ w9_t ^ w3_t ^ w1_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w1_t); - w2_t = rotl32 ((wf_t ^ wa_t ^ w4_t ^ w2_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w2_t); - w3_t = rotl32 ((w0_t ^ wb_t ^ w5_t ^ w3_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w3_t); - w4_t = rotl32 ((w1_t ^ wc_t ^ w6_t ^ w4_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w4_t); - w5_t = rotl32 ((w2_t ^ wd_t ^ w7_t ^ w5_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, w5_t); - w6_t = rotl32 ((w3_t ^ we_t ^ w8_t ^ w6_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, w6_t); - w7_t = rotl32 ((w4_t ^ wf_t ^ w9_t ^ w7_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, w7_t); - w8_t = rotl32 ((w5_t ^ w0_t ^ wa_t ^ w8_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, w8_t); - w9_t = rotl32 ((w6_t ^ w1_t ^ wb_t ^ w9_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, w9_t); - wa_t = rotl32 ((w7_t ^ w2_t ^ wc_t ^ wa_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wa_t); - wb_t = rotl32 ((w8_t ^ w3_t ^ wd_t ^ wb_t), 1u); SHA1_STEP (SHA1_F1, a, b, c, d, e, wb_t); - - if (MATCHES_NONE_VS (e, e_rev)) continue; - - wc_t = rotl32 ((w9_t ^ w4_t ^ we_t ^ wc_t), 1u); SHA1_STEP (SHA1_F1, e, a, b, c, d, wc_t); - wd_t = rotl32 ((wa_t ^ w5_t ^ wf_t ^ wd_t), 1u); SHA1_STEP (SHA1_F1, d, e, a, b, c, wd_t); - we_t = rotl32 ((wb_t ^ w6_t ^ w0_t ^ we_t), 1u); SHA1_STEP (SHA1_F1, c, d, e, a, b, we_t); - wf_t = rotl32 ((wc_t ^ w7_t ^ w1_t ^ wf_t), 1u); SHA1_STEP (SHA1_F1, b, c, d, e, a, wf_t); - - COMPARE_S_SIMD (d, e, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m00400-optimized.cl b/OpenCL/m00400-optimized.cl new file mode 100644 index 000000000..ddcb406a1 --- /dev/null +++ b/OpenCL/m00400-optimized.cl @@ -0,0 +1,425 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + + u32 w0_t = w0[0]; + u32 w1_t = w0[1]; + u32 w2_t = w0[2]; + u32 w3_t = w0[3]; + u32 w4_t = w1[0]; + u32 w5_t = w1[1]; + u32 w6_t = w1[2]; + u32 w7_t = w1[3]; + u32 w8_t = w2[0]; + u32 w9_t = w2[1]; + u32 wa_t = w2[2]; + u32 wb_t = w2[3]; + u32 wc_t = w3[0]; + u32 wd_t = w3[1]; + u32 we_t = w3[2]; + u32 wf_t = 0; + + MD5_STEP_S (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP_S (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP_S (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP_S (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP_S (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP_S (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP_S (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP_S (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP_S (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP_S (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP_S (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP_S (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP_S (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP_S (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP_S (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP_S (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP_S (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP_S (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP_S (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP_S (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP_S (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP_S (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP_S (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP_S (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP_S (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP_S (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP_S (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP_S (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP_S (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP_S (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP_S (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP_S (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP_S (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP_S (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP_S (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP_S (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP_S (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP_S (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP_S (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP_S (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP_S (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP_S (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP_S (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP_S (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP_S (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP_S (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP_S (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP_S (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP_S (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP_S (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP_S (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP_S (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP_S (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP_S (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP_S (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP_S (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP_S (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP_S (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP_S (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP_S (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP_S (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP_S (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP_S (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP_S (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void md5_transform_V (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = 0; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +__kernel void m00400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = 0; + w2[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + /** + * init + */ + + u32 block_len = 8 + pw_len; + + u32 block0[4]; + + block0[0] = salt_buf[0]; + block0[1] = salt_buf[1]; + block0[2] = w0[0]; + block0[3] = w0[1]; + + u32 block1[4]; + + block1[0] = w0[2]; + block1[1] = w0[3]; + block1[2] = w1[0]; + block1[3] = w1[1]; + + u32 block2[4]; + + block2[0] = w1[2]; + block2[1] = w1[3]; + block2[2] = w2[0]; + block2[3] = w2[1]; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = block_len * 8; + block3[3] = 0; + + append_0x80_4x4_S (block0, block1, block2, block3, block_len); + + /** + * init + */ + + u32 digest[4]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform_S (block0, block1, block2, block3, digest); + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m00400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if ((gid * VECT_SIZE) >= gid_max) return; + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + + w0[0] = packv (pws, i, gid, 0); + w0[1] = packv (pws, i, gid, 1); + w0[2] = packv (pws, i, gid, 2); + w0[3] = packv (pws, i, gid, 3); + w1[0] = packv (pws, i, gid, 4); + w1[1] = packv (pws, i, gid, 5); + w1[2] = packv (pws, i, gid, 6); + w1[3] = packv (pws, i, gid, 7); + w2[0] = packv (pws, i, gid, 8); + w2[1] = packv (pws, i, gid, 9); + w2[2] = 0; + w2[3] = 0; + + u32x pw_len = packvf (pws, pw_len, gid); + + u32x digest[4]; + + digest[0] = packv (tmps, digest_buf, gid, 0); + digest[1] = packv (tmps, digest_buf, gid, 1); + digest[2] = packv (tmps, digest_buf, gid, 2); + digest[3] = packv (tmps, digest_buf, gid, 3); + + /** + * loop + */ + + u32x block_len = (16 + pw_len); + + u32x block0[4]; + u32x block1[4]; + u32x block2[4]; + u32x block3[4]; + + block0[0] = 0; + block0[1] = 0; + block0[2] = 0; + block0[3] = 0; + block1[0] = w0[0]; + block1[1] = w0[1]; + block1[2] = w0[2]; + block1[3] = w0[3]; + block2[0] = w1[0]; + block2[1] = w1[1]; + block2[2] = w1[2]; + block2[3] = w1[3]; + block3[0] = w2[0]; + block3[1] = w2[1]; + block3[2] = block_len * 8; + block3[3] = 0; + + append_0x80_4x4_VV (block0, block1, block2, block3, block_len); + + /** + * init + */ + + for (u32 i = 0; i < loop_cnt; i++) + { + block0[0] = digest[0]; + block0[1] = digest[1]; + block0[2] = digest[2]; + block0[3] = digest[3]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform_V (block0, block1, block2, block3, digest); + } + + unpackv (tmps, digest_buf, gid, 0, digest[0]); + unpackv (tmps, digest_buf, gid, 1, digest[1]); + unpackv (tmps, digest_buf, gid, 2, digest[2]); + unpackv (tmps, digest_buf, gid, 3, digest[3]); +} + +__kernel void m00400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + if (gid >= gid_max) return; + + /** + * digest + */ + + const u32 r0 = tmps[gid].digest_buf[DGST_R0]; + const u32 r1 = tmps[gid].digest_buf[DGST_R1]; + const u32 r2 = tmps[gid].digest_buf[DGST_R2]; + const u32 r3 = tmps[gid].digest_buf[DGST_R3]; + + #define il_pos 0 + + #include COMPARE_M +} \ No newline at end of file diff --git a/OpenCL/m00400-pure.cl b/OpenCL/m00400-pure.cl deleted file mode 100644 index 821e4a02a..000000000 --- a/OpenCL/m00400-pure.cl +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -__kernel void m00400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - md5_update_global (&md5_ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_update_global (&md5_ctx, pws[gid].i, pws[gid].pw_len); - - md5_final (&md5_ctx); - - u32 digest[4]; - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; -} - -__kernel void m00400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - u32 digest[4]; - - digest[0] = tmps[gid].digest_buf[0]; - digest[1] = tmps[gid].digest_buf[1]; - digest[2] = tmps[gid].digest_buf[2]; - digest[3] = tmps[gid].digest_buf[3]; - - /** - * loop - */ - - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - md5_ctx.w0[0] = digest[0]; - md5_ctx.w0[1] = digest[1]; - md5_ctx.w0[2] = digest[2]; - md5_ctx.w0[3] = digest[3]; - - md5_ctx.len = 16; - - md5_update (&md5_ctx, w, pw_len); - - md5_final (&md5_ctx); - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - - if ((16 + pw_len + 1) >= 56) - { - for (u32 i = 1; i < loop_cnt; i++) - { - md5_init (&md5_ctx); - - md5_ctx.w0[0] = digest[0]; - md5_ctx.w0[1] = digest[1]; - md5_ctx.w0[2] = digest[2]; - md5_ctx.w0[3] = digest[3]; - - md5_ctx.len = 16; - - md5_update (&md5_ctx, w, pw_len); - - md5_final (&md5_ctx); - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - } - } - else - { - for (u32 i = 1; i < loop_cnt; i++) - { - md5_ctx.w0[0] = digest[0]; - md5_ctx.w0[1] = digest[1]; - md5_ctx.w0[2] = digest[2]; - md5_ctx.w0[3] = digest[3]; - - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform (md5_ctx.w0, md5_ctx.w1, md5_ctx.w2, md5_ctx.w3, digest); - } - } - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; -} - -__kernel void m00400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 r0 = tmps[gid].digest_buf[DGST_R0]; - const u32 r1 = tmps[gid].digest_buf[DGST_R1]; - const u32 r2 = tmps[gid].digest_buf[DGST_R2]; - const u32 r3 = tmps[gid].digest_buf[DGST_R3]; - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m00400.cl b/OpenCL/m00400.cl index ddcb406a1..821e4a02a 100644 --- a/OpenCL/m00400.cl +++ b/OpenCL/m00400.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -11,206 +11,11 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" -void md5_transform_S (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - - u32 w0_t = w0[0]; - u32 w1_t = w0[1]; - u32 w2_t = w0[2]; - u32 w3_t = w0[3]; - u32 w4_t = w1[0]; - u32 w5_t = w1[1]; - u32 w6_t = w1[2]; - u32 w7_t = w1[3]; - u32 w8_t = w2[0]; - u32 w9_t = w2[1]; - u32 wa_t = w2[2]; - u32 wb_t = w2[3]; - u32 wc_t = w3[0]; - u32 wd_t = w3[1]; - u32 we_t = w3[2]; - u32 wf_t = 0; - - MD5_STEP_S (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP_S (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP_S (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP_S (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP_S (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP_S (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP_S (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP_S (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP_S (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP_S (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP_S (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP_S (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP_S (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP_S (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP_S (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP_S (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP_S (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP_S (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP_S (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP_S (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP_S (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP_S (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP_S (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP_S (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP_S (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP_S (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP_S (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP_S (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP_S (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP_S (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP_S (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP_S (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP_S (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP_S (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP_S (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP_S (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP_S (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP_S (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP_S (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP_S (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP_S (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP_S (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP_S (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP_S (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP_S (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP_S (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP_S (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP_S (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP_S (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP_S (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP_S (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP_S (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP_S (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP_S (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP_S (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP_S (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP_S (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP_S (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP_S (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP_S (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP_S (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP_S (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP_S (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP_S (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void md5_transform_V (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[4]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = 0; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - __kernel void m00400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -221,86 +26,26 @@ __kernel void m00400_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = 0; - w2[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - /** * init */ - u32 block_len = 8 + pw_len; + md5_ctx_t md5_ctx; - u32 block0[4]; + md5_init (&md5_ctx); - block0[0] = salt_buf[0]; - block0[1] = salt_buf[1]; - block0[2] = w0[0]; - block0[3] = w0[1]; + md5_update_global (&md5_ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - u32 block1[4]; + md5_update_global (&md5_ctx, pws[gid].i, pws[gid].pw_len); - block1[0] = w0[2]; - block1[1] = w0[3]; - block1[2] = w1[0]; - block1[3] = w1[1]; - - u32 block2[4]; - - block2[0] = w1[2]; - block2[1] = w1[3]; - block2[2] = w2[0]; - block2[3] = w2[1]; - - u32 block3[4]; - - block3[0] = 0; - block3[1] = 0; - block3[2] = block_len * 8; - block3[3] = 0; - - append_0x80_4x4_S (block0, block1, block2, block3, block_len); - - /** - * init - */ + md5_final (&md5_ctx); u32 digest[4]; - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform_S (block0, block1, block2, block3, digest); + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; tmps[gid].digest_buf[0] = digest[0]; tmps[gid].digest_buf[1] = digest[1]; @@ -316,87 +61,101 @@ __kernel void m00400_loop (__global pw_t *pws, __global const kernel_rule_t *rul const u32 gid = get_global_id (0); - if ((gid * VECT_SIZE) >= gid_max) return; - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - - w0[0] = packv (pws, i, gid, 0); - w0[1] = packv (pws, i, gid, 1); - w0[2] = packv (pws, i, gid, 2); - w0[3] = packv (pws, i, gid, 3); - w1[0] = packv (pws, i, gid, 4); - w1[1] = packv (pws, i, gid, 5); - w1[2] = packv (pws, i, gid, 6); - w1[3] = packv (pws, i, gid, 7); - w2[0] = packv (pws, i, gid, 8); - w2[1] = packv (pws, i, gid, 9); - w2[2] = 0; - w2[3] = 0; - - u32x pw_len = packvf (pws, pw_len, gid); - - u32x digest[4]; - - digest[0] = packv (tmps, digest_buf, gid, 0); - digest[1] = packv (tmps, digest_buf, gid, 1); - digest[2] = packv (tmps, digest_buf, gid, 2); - digest[3] = packv (tmps, digest_buf, gid, 3); - - /** - * loop - */ - - u32x block_len = (16 + pw_len); - - u32x block0[4]; - u32x block1[4]; - u32x block2[4]; - u32x block3[4]; - - block0[0] = 0; - block0[1] = 0; - block0[2] = 0; - block0[3] = 0; - block1[0] = w0[0]; - block1[1] = w0[1]; - block1[2] = w0[2]; - block1[3] = w0[3]; - block2[0] = w1[0]; - block2[1] = w1[1]; - block2[2] = w1[2]; - block2[3] = w1[3]; - block3[0] = w2[0]; - block3[1] = w2[1]; - block3[2] = block_len * 8; - block3[3] = 0; - - append_0x80_4x4_VV (block0, block1, block2, block3, block_len); + if (gid >= gid_max) return; /** * init */ - for (u32 i = 0; i < loop_cnt; i++) + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) { - block0[0] = digest[0]; - block0[1] = digest[1]; - block0[2] = digest[2]; - block0[3] = digest[3]; + w[idx] = pws[gid].i[idx]; - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform_V (block0, block1, block2, block3, digest); + barrier (CLK_GLOBAL_MEM_FENCE); } - unpackv (tmps, digest_buf, gid, 0, digest[0]); - unpackv (tmps, digest_buf, gid, 1, digest[1]); - unpackv (tmps, digest_buf, gid, 2, digest[2]); - unpackv (tmps, digest_buf, gid, 3, digest[3]); + u32 digest[4]; + + digest[0] = tmps[gid].digest_buf[0]; + digest[1] = tmps[gid].digest_buf[1]; + digest[2] = tmps[gid].digest_buf[2]; + digest[3] = tmps[gid].digest_buf[3]; + + /** + * loop + */ + + md5_ctx_t md5_ctx; + + md5_init (&md5_ctx); + + md5_ctx.w0[0] = digest[0]; + md5_ctx.w0[1] = digest[1]; + md5_ctx.w0[2] = digest[2]; + md5_ctx.w0[3] = digest[3]; + + md5_ctx.len = 16; + + md5_update (&md5_ctx, w, pw_len); + + md5_final (&md5_ctx); + + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; + + if ((16 + pw_len + 1) >= 56) + { + for (u32 i = 1; i < loop_cnt; i++) + { + md5_init (&md5_ctx); + + md5_ctx.w0[0] = digest[0]; + md5_ctx.w0[1] = digest[1]; + md5_ctx.w0[2] = digest[2]; + md5_ctx.w0[3] = digest[3]; + + md5_ctx.len = 16; + + md5_update (&md5_ctx, w, pw_len); + + md5_final (&md5_ctx); + + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; + } + } + else + { + for (u32 i = 1; i < loop_cnt; i++) + { + md5_ctx.w0[0] = digest[0]; + md5_ctx.w0[1] = digest[1]; + md5_ctx.w0[2] = digest[2]; + md5_ctx.w0[3] = digest[3]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (md5_ctx.w0, md5_ctx.w1, md5_ctx.w2, md5_ctx.w3, digest); + } + } + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; } __kernel void m00400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global phpass_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -422,4 +181,4 @@ __kernel void m00400_comp (__global pw_t *pws, __global const kernel_rule_t *rul #define il_pos 0 #include COMPARE_M -} \ No newline at end of file +} diff --git a/OpenCL/m00500-optimized.cl b/OpenCL/m00500-optimized.cl new file mode 100644 index 000000000..81cd4f8bf --- /dev/null +++ b/OpenCL/m00500-optimized.cl @@ -0,0 +1,1124 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +//#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +#define md5crypt_magic 0x00243124u + +void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + + u32 w0_t = w0[0]; + u32 w1_t = w0[1]; + u32 w2_t = w0[2]; + u32 w3_t = w0[3]; + u32 w4_t = w1[0]; + u32 w5_t = w1[1]; + u32 w6_t = w1[2]; + u32 w7_t = w1[3]; + u32 w8_t = w2[0]; + u32 w9_t = w2[1]; + u32 wa_t = w2[2]; + u32 wb_t = w2[3]; + u32 wc_t = w3[0]; + u32 wd_t = w3[1]; + u32 we_t = w3[2]; + u32 wf_t = 0; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0x80; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0x80, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + break; + case 10: block2[2] |= tmp0; + block2[3] = tmp1; + block3[0] = tmp2; + break; + case 11: block2[3] |= tmp0; + block3[0] = tmp1; + block3[1] = tmp2; + break; + } +} + +void append_sign (u32 block0[4], u32 block1[4], const u32 block_len) +{ + switch (block_len) + { + case 0: + block0[0] = md5crypt_magic; + break; + + case 1: + block0[0] = block0[0] | md5crypt_magic << 8u; + block0[1] = md5crypt_magic >> 24u; + break; + + case 2: + block0[0] = block0[0] | md5crypt_magic << 16u; + block0[1] = md5crypt_magic >> 16u; + break; + + case 3: + block0[0] = block0[0] | md5crypt_magic << 24u; + block0[1] = md5crypt_magic >> 8u; + break; + + case 4: + block0[1] = md5crypt_magic; + break; + + case 5: + block0[1] = block0[1] | md5crypt_magic << 8u; + block0[2] = md5crypt_magic >> 24u; + break; + + case 6: + block0[1] = block0[1] | md5crypt_magic << 16u; + block0[2] = md5crypt_magic >> 16u; + break; + + case 7: + block0[1] = block0[1] | md5crypt_magic << 24u; + block0[2] = md5crypt_magic >> 8u; + break; + + case 8: + block0[2] = md5crypt_magic; + break; + + case 9: + block0[2] = block0[2] | md5crypt_magic << 8u; + block0[3] = md5crypt_magic >> 24u; + break; + + case 10: + block0[2] = block0[2] | md5crypt_magic << 16u; + block0[3] = md5crypt_magic >> 16u; + break; + + case 11: + block0[2] = block0[2] | md5crypt_magic << 24u; + block0[3] = md5crypt_magic >> 8u; + break; + + case 12: + block0[3] = md5crypt_magic; + break; + + case 13: + block0[3] = block0[3] | md5crypt_magic << 8u; + block1[0] = md5crypt_magic >> 24u; + break; + + case 14: + block0[3] = block0[3] | md5crypt_magic << 16u; + block1[0] = md5crypt_magic >> 16u; + break; + + case 15: + block0[3] = block0[3] | md5crypt_magic << 24u; + block1[0] = md5crypt_magic >> 8u; + break; + } +} + +void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) +{ + switch (block_len) + { + case 0: + block0[0] = append; + break; + + case 1: + block0[0] = block0[0] | append << 8; + break; + + case 2: + block0[0] = block0[0] | append << 16; + break; + + case 3: + block0[0] = block0[0] | append << 24; + break; + + case 4: + block0[1] = append; + break; + + case 5: + block0[1] = block0[1] | append << 8; + break; + + case 6: + block0[1] = block0[1] | append << 16; + break; + + case 7: + block0[1] = block0[1] | append << 24; + break; + + case 8: + block0[2] = append; + break; + + case 9: + block0[2] = block0[2] | append << 8; + break; + + case 10: + block0[2] = block0[2] | append << 16; + break; + + case 11: + block0[2] = block0[2] | append << 24; + break; + + case 12: + block0[3] = append; + break; + + case 13: + block0[3] = block0[3] | append << 8; + break; + + case 14: + block0[3] = block0[3] | append << 16; + break; + + case 15: + block0[3] = block0[3] | append << 24; + break; + + case 16: + block1[0] = append; + break; + + case 17: + block1[0] = block1[0] | append << 8; + break; + + case 18: + block1[0] = block1[0] | append << 16; + break; + + case 19: + block1[0] = block1[0] | append << 24; + break; + + case 20: + block1[1] = append; + break; + + case 21: + block1[1] = block1[1] | append << 8; + break; + + case 22: + block1[1] = block1[1] | append << 16; + break; + + case 23: + block1[1] = block1[1] | append << 24; + break; + + case 24: + block1[2] = append; + break; + + case 25: + block1[2] = block1[2] | append << 8; + break; + + case 26: + block1[2] = block1[2] | append << 16; + break; + + case 27: + block1[2] = block1[2] | append << 24; + break; + + case 28: + block1[3] = append; + break; + + case 29: + block1[3] = block1[3] | append << 8; + break; + + case 30: + block1[3] = block1[3] | append << 16; + break; + + case 31: + block1[3] = block1[3] | append << 24; + break; + + case 32: + block2[0] = append; + break; + + case 33: + block2[0] = block2[0] | append << 8; + break; + + case 34: + block2[0] = block2[0] | append << 16; + break; + + case 35: + block2[0] = block2[0] | append << 24; + break; + + case 36: + block2[1] = append; + break; + + case 37: + block2[1] = block2[1] | append << 8; + break; + + case 38: + block2[1] = block2[1] | append << 16; + break; + + case 39: + block2[1] = block2[1] | append << 24; + break; + + case 40: + block2[2] = append; + break; + + case 41: + block2[2] = block2[2] | append << 8; + break; + + case 42: + block2[2] = block2[2] | append << 16; + break; + + case 43: + block2[2] = block2[2] | append << 24; + break; + + case 44: + block2[3] = append; + break; + + case 45: + block2[3] = block2[3] | append << 8; + break; + + case 46: + block2[3] = block2[3] | append << 16; + break; + + case 47: + block2[3] = block2[3] | append << 24; + break; + + case 48: + block3[0] = append; + break; + + case 49: + block3[0] = block3[0] | append << 8; + break; + + case 50: + block3[0] = block3[0] | append << 16; + break; + + case 51: + block3[0] = block3[0] | append << 24; + break; + + case 52: + block3[1] = append; + break; + + case 53: + block3[1] = block3[1] | append << 8; + break; + + case 54: + block3[1] = block3[1] | append << 16; + break; + + case 55: + block3[1] = block3[1] | append << 24; + break; + + case 56: + block3[2] = append; + break; + } +} + +__kernel void m00500_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * init + */ + + //memcat16 (block0, block1, block2, block3, block_len, w0); + //block_len += pw_len; + + u32 block_len = pw_len; + + u32 block0[4]; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + u32 digest[4]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + /* The password first, since that is what is most unknown */ + /* Then our magic string */ + /* Then the raw salt */ + /* Then just as many characters of the MD5(pw,salt,pw) */ + + //memcat16 (block0, block1, block2, block3, block_len, w); + //block_len += pw_len; + + block_len = pw_len; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + append_sign (block0, block1, block_len); + + block_len += 3; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + truncate_block_4x4_le (digest, pw_len); + + memcat16 (block0, block1, block2, block3, block_len, digest); + + block_len += pw_len; + + /* Then something really weird... */ + + u32 append = block0[0] & 0xFF; + + for (u32 j = pw_len; j; j >>= 1) + { + if ((j & 1) == 0) + { + append_1st (block0, block1, block2, block3, block_len, append); + } + + block_len++; + } + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m00500_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + u32 w0_x80[4]; + + w0_x80[0] = w0[0]; + w0_x80[1] = w0[1]; + w0_x80[2] = w0[2]; + w0_x80[3] = w0[3]; + + append_0x80_1x4 (w0_x80, pw_len); + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + u32 digest[4]; + + digest[0] = tmps[gid].digest_buf[0]; + digest[1] = tmps[gid].digest_buf[1]; + digest[2] = tmps[gid].digest_buf[2]; + digest[3] = tmps[gid].digest_buf[3]; + + /** + * loop + */ + + /* and now, just to make sure things don't run too fast */ + + u32 block_len; + + u32 block0[4]; + + block0[0] = 0; + block0[1] = 0; + block0[2] = 0; + block0[3] = 0; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + block3[0] = 0; + block3[1] = 0; + + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 1 : 0; + const u32 j7 = (j % 7) ? 1 : 0; + + if (j1) + { + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block_len = pw_len; + + if (j3) + { + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + } + + if (j7) + { + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + + memcat16_x80 (block0, block1, block2, block3, block_len, digest); + + block_len += 16; + } + else + { + block0[0] = digest[0]; + block0[1] = digest[1]; + block0[2] = digest[2]; + block0[3] = digest[3]; + + block_len = 16; + + if (j3 && j7) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + else if (j3) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + } + else if (j7) + { + block1[0] = w0[0]; + block1[1] = w0[1]; + block1[2] = w0[2]; + block1[3] = w0[3]; + + block_len += pw_len; + } + + memcat16 (block0, block1, block2, block3, block_len, w0_x80); + + block_len += pw_len; + } + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + } + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m00500_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 r0 = tmps[gid].digest_buf[DGST_R0]; + const u32 r1 = tmps[gid].digest_buf[DGST_R1]; + const u32 r2 = tmps[gid].digest_buf[DGST_R2]; + const u32 r3 = tmps[gid].digest_buf[DGST_R3]; + + #define il_pos 0 + + #include COMPARE_M +} diff --git a/OpenCL/m00500-pure.cl b/OpenCL/m00500-pure.cl deleted file mode 100644 index 54422ec96..000000000 --- a/OpenCL/m00500-pure.cl +++ /dev/null @@ -1,261 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_md5.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -#define md5crypt_magic 0x00243124u - -__kernel void m00500_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * prepare - */ - - md5_ctx_t md5_ctx1; - - md5_init (&md5_ctx1); - - md5_update (&md5_ctx1, w, pw_len); - - md5_update (&md5_ctx1, s, salt_len); - - md5_update (&md5_ctx1, w, pw_len); - - md5_final (&md5_ctx1); - - u32 final[16] = { 0 }; - - final[0] = md5_ctx1.h[0]; - final[1] = md5_ctx1.h[1]; - final[2] = md5_ctx1.h[2]; - final[3] = md5_ctx1.h[3]; - - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - md5_update (&md5_ctx, w, pw_len); - - u32 m[16] = { 0 }; - - m[0] = md5crypt_magic; - - md5_update (&md5_ctx, m, 3); - - md5_update (&md5_ctx, s, salt_len); - - int pl; - - for (pl = pw_len; pl > 16; pl -= 16) - { - md5_update (&md5_ctx, final, 16); - } - - truncate_block_4x4_le (final, pl); - - md5_update (&md5_ctx, final, pl); - - /* Then something really weird... */ - - for (int i = pw_len; i != 0; i >>= 1) - { - u32 t[16] = { 0 }; - - if (i & 1) - { - t[0] = 0; - } - else - { - t[0] = w[0] & 0xff; - } - - md5_update (&md5_ctx, t, 1); - } - - md5_final (&md5_ctx); - - tmps[gid].digest_buf[0] = md5_ctx.h[0]; - tmps[gid].digest_buf[1] = md5_ctx.h[1]; - tmps[gid].digest_buf[2] = md5_ctx.h[2]; - tmps[gid].digest_buf[3] = md5_ctx.h[3]; -} - -__kernel void m00500_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * digest - */ - - u32 digest[16] = { 0 }; - - digest[0] = tmps[gid].digest_buf[0]; - digest[1] = tmps[gid].digest_buf[1]; - digest[2] = tmps[gid].digest_buf[2]; - digest[3] = tmps[gid].digest_buf[3]; - - /** - * loop - */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - if (j & 1) - { - md5_update (&md5_ctx, w, pw_len); - } - else - { - md5_update (&md5_ctx, digest, 16); - } - - if (j % 3) - { - md5_update (&md5_ctx, s, salt_len); - } - - if (j % 7) - { - md5_update (&md5_ctx, w, pw_len); - } - - if (j & 1) - { - md5_update (&md5_ctx, digest, 16); - } - else - { - md5_update (&md5_ctx, w, pw_len); - } - - md5_final (&md5_ctx); - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - } - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; -} - -__kernel void m00500_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - /** - * digest - */ - - const u32 r0 = tmps[gid].digest_buf[DGST_R0]; - const u32 r1 = tmps[gid].digest_buf[DGST_R1]; - const u32 r2 = tmps[gid].digest_buf[DGST_R2]; - const u32 r3 = tmps[gid].digest_buf[DGST_R3]; - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m00500.cl b/OpenCL/m00500.cl index 81cd4f8bf..54422ec96 100644 --- a/OpenCL/m00500.cl +++ b/OpenCL/m00500.cl @@ -10,742 +10,13 @@ #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_md5.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" #define md5crypt_magic 0x00243124u -void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - - u32 w0_t = w0[0]; - u32 w1_t = w0[1]; - u32 w2_t = w0[2]; - u32 w3_t = w0[3]; - u32 w4_t = w1[0]; - u32 w5_t = w1[1]; - u32 w6_t = w1[2]; - u32 w7_t = w1[3]; - u32 w8_t = w2[0]; - u32 w9_t = w2[1]; - u32 wa_t = w2[2]; - u32 wb_t = w2[3]; - u32 wc_t = w3[0]; - u32 wd_t = w3[1]; - u32 we_t = w3[2]; - u32 wf_t = 0; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0x80; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0x80, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - break; - case 10: block2[2] |= tmp0; - block2[3] = tmp1; - block3[0] = tmp2; - break; - case 11: block2[3] |= tmp0; - block3[0] = tmp1; - block3[1] = tmp2; - break; - } -} - -void append_sign (u32 block0[4], u32 block1[4], const u32 block_len) -{ - switch (block_len) - { - case 0: - block0[0] = md5crypt_magic; - break; - - case 1: - block0[0] = block0[0] | md5crypt_magic << 8u; - block0[1] = md5crypt_magic >> 24u; - break; - - case 2: - block0[0] = block0[0] | md5crypt_magic << 16u; - block0[1] = md5crypt_magic >> 16u; - break; - - case 3: - block0[0] = block0[0] | md5crypt_magic << 24u; - block0[1] = md5crypt_magic >> 8u; - break; - - case 4: - block0[1] = md5crypt_magic; - break; - - case 5: - block0[1] = block0[1] | md5crypt_magic << 8u; - block0[2] = md5crypt_magic >> 24u; - break; - - case 6: - block0[1] = block0[1] | md5crypt_magic << 16u; - block0[2] = md5crypt_magic >> 16u; - break; - - case 7: - block0[1] = block0[1] | md5crypt_magic << 24u; - block0[2] = md5crypt_magic >> 8u; - break; - - case 8: - block0[2] = md5crypt_magic; - break; - - case 9: - block0[2] = block0[2] | md5crypt_magic << 8u; - block0[3] = md5crypt_magic >> 24u; - break; - - case 10: - block0[2] = block0[2] | md5crypt_magic << 16u; - block0[3] = md5crypt_magic >> 16u; - break; - - case 11: - block0[2] = block0[2] | md5crypt_magic << 24u; - block0[3] = md5crypt_magic >> 8u; - break; - - case 12: - block0[3] = md5crypt_magic; - break; - - case 13: - block0[3] = block0[3] | md5crypt_magic << 8u; - block1[0] = md5crypt_magic >> 24u; - break; - - case 14: - block0[3] = block0[3] | md5crypt_magic << 16u; - block1[0] = md5crypt_magic >> 16u; - break; - - case 15: - block0[3] = block0[3] | md5crypt_magic << 24u; - block1[0] = md5crypt_magic >> 8u; - break; - } -} - -void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) -{ - switch (block_len) - { - case 0: - block0[0] = append; - break; - - case 1: - block0[0] = block0[0] | append << 8; - break; - - case 2: - block0[0] = block0[0] | append << 16; - break; - - case 3: - block0[0] = block0[0] | append << 24; - break; - - case 4: - block0[1] = append; - break; - - case 5: - block0[1] = block0[1] | append << 8; - break; - - case 6: - block0[1] = block0[1] | append << 16; - break; - - case 7: - block0[1] = block0[1] | append << 24; - break; - - case 8: - block0[2] = append; - break; - - case 9: - block0[2] = block0[2] | append << 8; - break; - - case 10: - block0[2] = block0[2] | append << 16; - break; - - case 11: - block0[2] = block0[2] | append << 24; - break; - - case 12: - block0[3] = append; - break; - - case 13: - block0[3] = block0[3] | append << 8; - break; - - case 14: - block0[3] = block0[3] | append << 16; - break; - - case 15: - block0[3] = block0[3] | append << 24; - break; - - case 16: - block1[0] = append; - break; - - case 17: - block1[0] = block1[0] | append << 8; - break; - - case 18: - block1[0] = block1[0] | append << 16; - break; - - case 19: - block1[0] = block1[0] | append << 24; - break; - - case 20: - block1[1] = append; - break; - - case 21: - block1[1] = block1[1] | append << 8; - break; - - case 22: - block1[1] = block1[1] | append << 16; - break; - - case 23: - block1[1] = block1[1] | append << 24; - break; - - case 24: - block1[2] = append; - break; - - case 25: - block1[2] = block1[2] | append << 8; - break; - - case 26: - block1[2] = block1[2] | append << 16; - break; - - case 27: - block1[2] = block1[2] | append << 24; - break; - - case 28: - block1[3] = append; - break; - - case 29: - block1[3] = block1[3] | append << 8; - break; - - case 30: - block1[3] = block1[3] | append << 16; - break; - - case 31: - block1[3] = block1[3] | append << 24; - break; - - case 32: - block2[0] = append; - break; - - case 33: - block2[0] = block2[0] | append << 8; - break; - - case 34: - block2[0] = block2[0] | append << 16; - break; - - case 35: - block2[0] = block2[0] | append << 24; - break; - - case 36: - block2[1] = append; - break; - - case 37: - block2[1] = block2[1] | append << 8; - break; - - case 38: - block2[1] = block2[1] | append << 16; - break; - - case 39: - block2[1] = block2[1] | append << 24; - break; - - case 40: - block2[2] = append; - break; - - case 41: - block2[2] = block2[2] | append << 8; - break; - - case 42: - block2[2] = block2[2] | append << 16; - break; - - case 43: - block2[2] = block2[2] | append << 24; - break; - - case 44: - block2[3] = append; - break; - - case 45: - block2[3] = block2[3] | append << 8; - break; - - case 46: - block2[3] = block2[3] | append << 16; - break; - - case 47: - block2[3] = block2[3] | append << 24; - break; - - case 48: - block3[0] = append; - break; - - case 49: - block3[0] = block3[0] | append << 8; - break; - - case 50: - block3[0] = block3[0] | append << 16; - break; - - case 51: - block3[0] = block3[0] | append << 24; - break; - - case 52: - block3[1] = append; - break; - - case 53: - block3[1] = block3[1] | append << 8; - break; - - case 54: - block3[1] = block3[1] | append << 16; - break; - - case 55: - block3[1] = block3[1] | append << 24; - break; - - case 56: - block3[2] = append; - break; - } -} - __kernel void m00500_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -756,157 +27,108 @@ __kernel void m00500_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * init */ - //memcat16 (block0, block1, block2, block3, block_len, w0); - //block_len += pw_len; + const u32 pw_len = pws[gid].pw_len; - u32 block_len = pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 block0[4]; + u32 w[64] = { 0 }; - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - u32 block1[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 block2[4]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + u32 s[64] = { 0 }; - u32 block3[4]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + barrier (CLK_GLOBAL_MEM_FENCE); + } - memcat8 (block0, block1, block2, block3, block_len, salt_buf); + /** + * prepare + */ - block_len += salt_len; + md5_ctx_t md5_ctx1; - memcat16 (block0, block1, block2, block3, block_len, w0); + md5_init (&md5_ctx1); - block_len += pw_len; + md5_update (&md5_ctx1, w, pw_len); - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_update (&md5_ctx1, s, salt_len); - block3[2] = block_len * 8; + md5_update (&md5_ctx1, w, pw_len); - u32 digest[4]; + md5_final (&md5_ctx1); - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + u32 final[16] = { 0 }; - md5_transform (block0, block1, block2, block3, digest); + final[0] = md5_ctx1.h[0]; + final[1] = md5_ctx1.h[1]; + final[2] = md5_ctx1.h[2]; + final[3] = md5_ctx1.h[3]; - /* The password first, since that is what is most unknown */ - /* Then our magic string */ - /* Then the raw salt */ - /* Then just as many characters of the MD5(pw,salt,pw) */ + md5_ctx_t md5_ctx; - //memcat16 (block0, block1, block2, block3, block_len, w); - //block_len += pw_len; + md5_init (&md5_ctx); - block_len = pw_len; + md5_update (&md5_ctx, w, pw_len); - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + u32 m[16] = { 0 }; - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + m[0] = md5crypt_magic; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + md5_update (&md5_ctx, m, 3); - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + md5_update (&md5_ctx, s, salt_len); - append_sign (block0, block1, block_len); + int pl; - block_len += 3; + for (pl = pw_len; pl > 16; pl -= 16) + { + md5_update (&md5_ctx, final, 16); + } - memcat8 (block0, block1, block2, block3, block_len, salt_buf); + truncate_block_4x4_le (final, pl); - block_len += salt_len; - - truncate_block_4x4_le (digest, pw_len); - - memcat16 (block0, block1, block2, block3, block_len, digest); - - block_len += pw_len; + md5_update (&md5_ctx, final, pl); /* Then something really weird... */ - u32 append = block0[0] & 0xFF; - - for (u32 j = pw_len; j; j >>= 1) + for (int i = pw_len; i != 0; i >>= 1) { - if ((j & 1) == 0) + u32 t[16] = { 0 }; + + if (i & 1) { - append_1st (block0, block1, block2, block3, block_len, append); + t[0] = 0; + } + else + { + t[0] = w[0] & 0xff; } - block_len++; + md5_update (&md5_ctx, t, 1); } - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_final (&md5_ctx); - block3[2] = block_len * 8; - - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform (block0, block1, block2, block3, digest); - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; + tmps[gid].digest_buf[0] = md5_ctx.h[0]; + tmps[gid].digest_buf[1] = md5_ctx.h[1]; + tmps[gid].digest_buf[2] = md5_ctx.h[2]; + tmps[gid].digest_buf[3] = md5_ctx.h[3]; } __kernel void m00500_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -919,40 +141,41 @@ __kernel void m00500_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; + /** + * init + */ const u32 pw_len = pws[gid].pw_len; - u32 w0_x80[4]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - w0_x80[0] = w0[0]; - w0_x80[1] = w0[1]; - w0_x80[2] = w0[2]; - w0_x80[3] = w0[3]; + u32 w[64] = { 0 }; - append_0x80_1x4 (w0_x80, pw_len); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * digest */ - u32 digest[4]; + u32 digest[16] = { 0 }; digest[0] = tmps[gid].digest_buf[0]; digest[1] = tmps[gid].digest_buf[1]; @@ -963,132 +186,46 @@ __kernel void m00500_loop (__global pw_t *pws, __global const kernel_rule_t *rul * loop */ - /* and now, just to make sure things don't run too fast */ - - u32 block_len; - - u32 block0[4]; - - block0[0] = 0; - block0[1] = 0; - block0[2] = 0; - block0[3] = 0; - - u32 block1[4]; - - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - - u32 block2[4]; - - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - - u32 block3[4]; - - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - block3[0] = 0; - block3[1] = 0; + md5_ctx_t md5_ctx; - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 1 : 0; - const u32 j7 = (j % 7) ? 1 : 0; + md5_init (&md5_ctx); - if (j1) + if (j & 1) { - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; - - block_len = pw_len; - - if (j3) - { - memcat8 (block0, block1, block2, block3, block_len, salt_buf); - - block_len += salt_len; - } - - if (j7) - { - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - - memcat16_x80 (block0, block1, block2, block3, block_len, digest); - - block_len += 16; + md5_update (&md5_ctx, w, pw_len); } - else + else { - block0[0] = digest[0]; - block0[1] = digest[1]; - block0[2] = digest[2]; - block0[3] = digest[3]; - - block_len = 16; - - if (j3 && j7) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - else if (j3) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - } - else if (j7) - { - block1[0] = w0[0]; - block1[1] = w0[1]; - block1[2] = w0[2]; - block1[3] = w0[3]; - - block_len += pw_len; - } - - memcat16 (block0, block1, block2, block3, block_len, w0_x80); - - block_len += pw_len; + md5_update (&md5_ctx, digest, 16); } - block3[2] = block_len * 8; + if (j % 3) + { + md5_update (&md5_ctx, s, salt_len); + } - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + if (j % 7) + { + md5_update (&md5_ctx, w, pw_len); + } - md5_transform (block0, block1, block2, block3, digest); + if (j & 1) + { + md5_update (&md5_ctx, digest, 16); + } + else + { + md5_update (&md5_ctx, w, pw_len); + } + + md5_final (&md5_ctx); + + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; } tmps[gid].digest_buf[0] = digest[0]; diff --git a/OpenCL/m00900_a0-optimized.cl b/OpenCL/m00900_a0-optimized.cl new file mode 100644 index 000000000..a62b6be8e --- /dev/null +++ b/OpenCL/m00900_a0-optimized.cl @@ -0,0 +1,262 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00900_a0-pure.cl b/OpenCL/m00900_a0-pure.cl deleted file mode 100644 index cf19e1c05..000000000 --- a/OpenCL/m00900_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx; - - md4_init (&ctx); - - md4_update (&ctx, w, pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx; - - md4_init (&ctx); - - md4_update (&ctx, w, pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00900_a0.cl b/OpenCL/m00900_a0.cl index a62b6be8e..cf19e1c05 100644 --- a/OpenCL/m00900_a0.cl +++ b/OpenCL/m00900_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,154 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx); - w3[2] = out_len * 8; - w3[3] = 0; + md4_update (&ctx, w, pw_len); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_final (&ctx); - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -172,91 +91,44 @@ __kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx); - w3[2] = out_len * 8; - w3[3] = 0; + md4_update (&ctx, w, pw_len); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_final (&ctx); - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00900_a1-optimized.cl b/OpenCL/m00900_a1-optimized.cl new file mode 100644 index 000000000..276fe0b61 --- /dev/null +++ b/OpenCL/m00900_a1-optimized.cl @@ -0,0 +1,378 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m00900_a1-pure.cl b/OpenCL/m00900_a1-pure.cl deleted file mode 100644 index 0bbd8e729..000000000 --- a/OpenCL/m00900_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx = ctx0; - - md4_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx = ctx0; - - md4_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00900_a1.cl b/OpenCL/m00900_a1.cl index 276fe0b61..0bbd8e729 100644 --- a/OpenCL/m00900_a1.cl +++ b/OpenCL/m00900_a1.cl @@ -3,220 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + md4_ctx_t ctx0; - if (gid >= gid_max) return; + md4_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + md4_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -229,150 +78,33 @@ __kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md4_ctx_t ctx0; + + md4_init (&ctx0); + + md4_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m00900_a3-optimized.cl b/OpenCL/m00900_a3-optimized.cl new file mode 100644 index 000000000..faa44690b --- /dev/null +++ b/OpenCL/m00900_a3-optimized.cl @@ -0,0 +1,580 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ +{ \ + a = rotr32_S (a, s); \ + a -= f (b, c, d); \ + a -= x; \ + a -= t; \ +} + +#define MD4_STEP_REV1(f,a,b,c,d,x,t,s) \ +{ \ + a = rotr32_S (a, s); \ + a -= x; \ + a -= t; \ +} + +void m00900m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m00900s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 7], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[11], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 3], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[13], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 5], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 9], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 1], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[14], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[10], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 2], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[12], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 4], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 8], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, 0, MD4C02, MD4S20); + + const u32 sav_c = c_rev; + const u32 sav_d = d_rev; + + MD4_STEP_REV1(MD4_G_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C01, MD4S13); + MD4_STEP_REV1(MD4_G_S, c_rev, d_rev, a_rev, b_rev, w[11], MD4C01, MD4S12); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x pre_a = a_rev; + u32x pre_b = b_rev; + u32x pre_c = c_rev; + + pre_a = pre_a - w0; + pre_b = pre_b - MD4_G (sav_c, sav_d, pre_a); + pre_c = pre_c - MD4_G (sav_d, pre_a, pre_b); + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); if (MATCHES_NONE_VV (c, pre_c)) continue; + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); if (MATCHES_NONE_VV (b, pre_b)) continue; + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); if (MATCHES_NONE_VV (a, pre_a)) continue; + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m00900_a3-pure.cl b/OpenCL/m00900_a3-pure.cl deleted file mode 100644 index 480a1e3c4..000000000 --- a/OpenCL/m00900_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md4.cl" - -__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - md4_update_vector (&ctx, w, pw_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - md4_update_vector (&ctx, w, pw_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m00900_a3.cl b/OpenCL/m00900_a3.cl index faa44690b..480a1e3c4 100644 --- a/OpenCL/m00900_a3.cl +++ b/OpenCL/m00900_a3.cl @@ -11,91 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md4.cl" -#define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ -{ \ - a = rotr32_S (a, s); \ - a -= f (b, c, d); \ - a -= x; \ - a -= t; \ -} - -#define MD4_STEP_REV1(f,a,b,c,d,x,t,s) \ -{ \ - a = rotr32_S (a, s); \ - a -= x; \ - a -= t; \ -} - -void m00900m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00900_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -103,129 +53,35 @@ void m00900m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + w[0] = w0; - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_ctx_vector_t ctx; - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + md4_init_vector (&ctx); - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + md4_update_vector (&ctx, w, pw_len); - COMPARE_M_SIMD (a, d, c, b); + md4_final_vector (&ctx); + + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m00900s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m00900_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; - - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; - - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + if (gid >= gid_max) return; /** * digest @@ -240,42 +96,27 @@ void m00900s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + const u32 pw_len = pws[gid].pw_len; - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 7], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[11], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 3], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[13], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 5], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 9], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 1], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[14], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[10], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 2], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[12], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 4], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 8], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, 0, MD4C02, MD4S20); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 sav_c = c_rev; - const u32 sav_d = d_rev; + u32x w[64] = { 0 }; - MD4_STEP_REV1(MD4_G_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C01, MD4S13); - MD4_STEP_REV1(MD4_G_S, c_rev, d_rev, a_rev, b_rev, w[11], MD4C01, MD4S12); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -283,298 +124,21 @@ void m00900s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x pre_a = a_rev; - u32x pre_b = b_rev; - u32x pre_c = c_rev; + w[0] = w0; - pre_a = pre_a - w0; - pre_b = pre_b - MD4_G (sav_c, sav_d, pre_a); - pre_c = pre_c - MD4_G (sav_d, pre_a, pre_b); + md4_ctx_vector_t ctx; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_init_vector (&ctx); - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_update_vector (&ctx, w, pw_len); - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); if (MATCHES_NONE_VV (c, pre_c)) continue; - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); if (MATCHES_NONE_VV (b, pre_b)) continue; - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); if (MATCHES_NONE_VV (a, pre_a)) continue; - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + md4_final_vector (&ctx); - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m00900_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00900_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00900_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00900_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00900_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m00900_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m00900s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01000_a0-optimized.cl b/OpenCL/m01000_a0-optimized.cl new file mode 100644 index 000000000..45b5866ff --- /dev/null +++ b/OpenCL/m01000_a0-optimized.cl @@ -0,0 +1,268 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = out_len * 8 * 2; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = out_len * 8 * 2; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01000_a0-pure.cl b/OpenCL/m01000_a0-pure.cl deleted file mode 100644 index d01ddc0d5..000000000 --- a/OpenCL/m01000_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx; - - md4_init (&ctx); - - md4_update_utf16le (&ctx, w, pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx; - - md4_init (&ctx); - - md4_update_utf16le (&ctx, w, pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01000_a0.cl b/OpenCL/m01000_a0.cl index 45b5866ff..d01ddc0d5 100644 --- a/OpenCL/m01000_a0.cl +++ b/OpenCL/m01000_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,157 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx); - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md4_update_utf16le (&ctx, w, pw_len); - w3[2] = out_len * 8 * 2; - w3[3] = 0; + md4_final (&ctx); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -175,94 +91,44 @@ __kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx); - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md4_update_utf16le (&ctx, w, pw_len); - w3[2] = out_len * 8 * 2; - w3[3] = 0; + md4_final (&ctx); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01000_a1-optimized.cl b/OpenCL/m01000_a1-optimized.cl new file mode 100644 index 000000000..d9db6192e --- /dev/null +++ b/OpenCL/m01000_a1-optimized.cl @@ -0,0 +1,390 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = pw_len * 8 * 2; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = pw_len * 8 * 2; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01000_a1-pure.cl b/OpenCL/m01000_a1-pure.cl deleted file mode 100644 index 03b6d08d3..000000000 --- a/OpenCL/m01000_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx = ctx0; - - md4_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx = ctx0; - - md4_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01000_a1.cl b/OpenCL/m01000_a1.cl index d9db6192e..03b6d08d3 100644 --- a/OpenCL/m01000_a1.cl +++ b/OpenCL/m01000_a1.cl @@ -3,226 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + md4_ctx_t ctx0; - if (gid >= gid_max) return; + md4_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - w3[2] = pw_len * 8 * 2; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -235,156 +78,33 @@ __kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md4_ctx_t ctx0; + + md4_init (&ctx0); + + md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global_utf16le (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - w3[2] = pw_len * 8 * 2; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01000_a3-optimized.cl b/OpenCL/m01000_a3-optimized.cl new file mode 100644 index 000000000..90fb7b79e --- /dev/null +++ b/OpenCL/m01000_a3-optimized.cl @@ -0,0 +1,586 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ +{ \ + a = rotr32_S (a, s); \ + a -= f (b, c, d); \ + a -= x; \ + a -= t; \ +} + +#define MD4_STEP_REV1(f,a,b,c,d,x,t,s) \ +{ \ + a = rotr32_S (a, s); \ + a -= x; \ + a -= t; \ +} + +void m01000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m01000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 7], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[11], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 3], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[13], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 5], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 9], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 1], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[14], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[10], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 2], MD4C02, MD4S20); + MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[12], MD4C02, MD4S23); + MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 4], MD4C02, MD4S22); + MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 8], MD4C02, MD4S21); + MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, 0, MD4C02, MD4S20); + + const u32 sav_c = c_rev; + const u32 sav_d = d_rev; + + MD4_STEP_REV1(MD4_G_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C01, MD4S13); + MD4_STEP_REV1(MD4_G_S, c_rev, d_rev, a_rev, b_rev, w[11], MD4C01, MD4S12); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x pre_a = a_rev; + u32x pre_b = b_rev; + u32x pre_c = c_rev; + + pre_a = pre_a - w0; + pre_b = pre_b - MD4_G (sav_c, sav_d, pre_a); + pre_c = pre_c - MD4_G (sav_d, pre_a, pre_b); + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + + if (MATCHES_NONE_VV (c, pre_c)) continue; + + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); + + if (MATCHES_NONE_VV (a, pre_a)) continue; + + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01000_a3-pure.cl b/OpenCL/m01000_a3-pure.cl deleted file mode 100644 index a9a421686..000000000 --- a/OpenCL/m01000_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md4.cl" - -__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - md4_update_vector_utf16le (&ctx, w, pw_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - md4_update_vector_utf16le (&ctx, w, pw_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01000_a3.cl b/OpenCL/m01000_a3.cl index 90fb7b79e..a9a421686 100644 --- a/OpenCL/m01000_a3.cl +++ b/OpenCL/m01000_a3.cl @@ -11,91 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md4.cl" -#define MD4_STEP_REV(f,a,b,c,d,x,t,s) \ -{ \ - a = rotr32_S (a, s); \ - a -= f (b, c, d); \ - a -= x; \ - a -= t; \ -} - -#define MD4_STEP_REV1(f,a,b,c,d,x,t,s) \ -{ \ - a = rotr32_S (a, s); \ - a -= x; \ - a -= t; \ -} - -void m01000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01000_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -103,129 +53,35 @@ void m01000m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + w[0] = w0; - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_ctx_vector_t ctx; - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + md4_init_vector (&ctx); - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + md4_update_vector_utf16le (&ctx, w, pw_len); - COMPARE_M_SIMD (a, d, c, b); + md4_final_vector (&ctx); + + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01000_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * base - */ - - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; - - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; - - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + if (gid >= gid_max) return; /** * digest @@ -240,42 +96,27 @@ void m01000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + const u32 pw_len = pws[gid].pw_len; - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 7], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[11], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 3], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[13], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 5], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 9], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 1], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[14], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 6], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[10], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, w[ 2], MD4C02, MD4S20); - MD4_STEP_REV (MD4_H_S, b_rev, c_rev, d_rev, a_rev, w[12], MD4C02, MD4S23); - MD4_STEP_REV (MD4_H_S, c_rev, d_rev, a_rev, b_rev, w[ 4], MD4C02, MD4S22); - MD4_STEP_REV (MD4_H_S, d_rev, a_rev, b_rev, c_rev, w[ 8], MD4C02, MD4S21); - MD4_STEP_REV (MD4_H_S, a_rev, b_rev, c_rev, d_rev, 0, MD4C02, MD4S20); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 sav_c = c_rev; - const u32 sav_d = d_rev; + u32x w[64] = { 0 }; - MD4_STEP_REV1(MD4_G_S, b_rev, c_rev, d_rev, a_rev, w[15], MD4C01, MD4S13); - MD4_STEP_REV1(MD4_G_S, c_rev, d_rev, a_rev, b_rev, w[11], MD4C01, MD4S12); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -283,304 +124,21 @@ void m01000s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x pre_a = a_rev; - u32x pre_b = b_rev; - u32x pre_c = c_rev; + w[0] = w0; - pre_a = pre_a - w0; - pre_b = pre_b - MD4_G (sav_c, sav_d, pre_a); - pre_c = pre_c - MD4_G (sav_d, pre_a, pre_b); + md4_ctx_vector_t ctx; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_init_vector (&ctx); - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_update_vector_utf16le (&ctx, w, pw_len); - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + md4_final_vector (&ctx); - if (MATCHES_NONE_VV (c, pre_c)) continue; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); - - if (MATCHES_NONE_VV (a, pre_a)) continue; - - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01000_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01000_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01000_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01000_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01000_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01000_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01000s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01100_a0-optimized.cl b/OpenCL/m01100_a0-optimized.cl new file mode 100644 index 000000000..4c7744058 --- /dev/null +++ b/OpenCL/m01100_a0-optimized.cl @@ -0,0 +1,468 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = out_len * 2 * 8; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + w0[0] = a; + w0[1] = b; + w0[2] = c; + w0[3] = d; + w1[0] = salt_buf00; + w1[1] = salt_buf01; + w1[2] = salt_buf02; + w1[3] = salt_buf03; + w2[0] = salt_buf04; + w2[1] = salt_buf05; + w2[2] = salt_buf06; + w2[3] = salt_buf07; + w3[0] = salt_buf08; + w3[1] = salt_buf09; + w3[2] = salt_buf10; + w3[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = out_len * 2 * 8; + w3[3] = 0; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + w0[0] = a; + w0[1] = b; + w0[2] = c; + w0[3] = d; + w1[0] = salt_buf00; + w1[1] = salt_buf01; + w1[2] = salt_buf02; + w1[3] = salt_buf03; + w2[0] = salt_buf04; + w2[1] = salt_buf05; + w2[2] = salt_buf06; + w2[3] = salt_buf07; + w3[0] = salt_buf08; + w3[1] = salt_buf09; + w3[2] = salt_buf10; + w3[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01100_a0-pure.cl b/OpenCL/m01100_a0-pure.cl deleted file mode 100644 index e257ed697..000000000 --- a/OpenCL/m01100_a0-pure.cl +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_utf16le (&ctx0, w, pw_len); - - md4_final (&ctx0); - - md4_ctx_t ctx; - - md4_init (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - - ctx.len = 16; - - md4_update_utf16le (&ctx, s, salt_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_utf16le (&ctx0, w, pw_len); - - md4_final (&ctx0); - - md4_ctx_t ctx; - - md4_init (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - - ctx.len = 16; - - md4_update_utf16le (&ctx, s, salt_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01100_a0.cl b/OpenCL/m01100_a0.cl index 4c7744058..e257ed697 100644 --- a/OpenCL/m01100_a0.cl +++ b/OpenCL/m01100_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,279 +12,101 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { + /** + * modifier + */ + + const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + /** * base */ - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - __local salt_t s_salt_buf[1]; + u32 w[64] = { 0 }; - if (lid == 0) + for (int idx = 0; idx < pw_lenv; idx++) { - s_salt_buf[0] = salt_bufs[salt_pos]; + w[idx] = pws[gid].i[idx]; - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + barrier (CLK_GLOBAL_MEM_FENCE); } - barrier (CLK_LOCAL_MEM_FENCE); + const u32 salt_len = salt_bufs[salt_pos].salt_len; - if (gid >= gid_max) return; + const u32 salt_lenv = ceil ((float) salt_len / 4); - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx0); - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md4_update_utf16le (&ctx0, w, pw_len); - w3[2] = out_len * 2 * 8; - w3[3] = 0; + md4_final (&ctx0); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_ctx_t ctx; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + md4_init (&ctx); - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + ctx.len = 16; - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; + md4_update_utf16le (&ctx, s, salt_len); - w0[0] = a; - w0[1] = b; - w0[2] = c; - w0[3] = d; - w1[0] = salt_buf00; - w1[1] = salt_buf01; - w1[2] = salt_buf02; - w1[3] = salt_buf03; - w2[0] = salt_buf04; - w2[1] = salt_buf05; - w2[2] = salt_buf06; - w2[3] = salt_buf07; - w3[0] = salt_buf08; - w3[1] = salt_buf09; - w3[2] = salt_buf10; - w3[3] = 0; + md4_final (&ctx); - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** - * base + * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); + const u32 gid = get_global_id (0); if (gid >= gid_max) return; - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] - /** * digest */ @@ -297,172 +119,72 @@ __kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md4_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md4_init (&ctx0); - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + md4_update_utf16le (&ctx0, w, pw_len); - w3[2] = out_len * 2 * 8; - w3[3] = 0; + md4_final (&ctx0); - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + md4_ctx_t ctx; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + md4_init (&ctx); - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + ctx.len = 16; - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; + md4_update_utf16le (&ctx, s, salt_len); - w0[0] = a; - w0[1] = b; - w0[2] = c; - w0[3] = d; - w1[0] = salt_buf00; - w1[1] = salt_buf01; - w1[2] = salt_buf02; - w1[3] = salt_buf03; - w2[0] = salt_buf04; - w2[1] = salt_buf05; - w2[2] = salt_buf06; - w2[3] = salt_buf07; - w3[0] = salt_buf08; - w3[1] = salt_buf09; - w3[2] = salt_buf10; - w3[3] = 0; + md4_final (&ctx); - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01100_a1-optimized.cl b/OpenCL/m01100_a1-optimized.cl new file mode 100644 index 000000000..bbb36149f --- /dev/null +++ b/OpenCL/m01100_a1-optimized.cl @@ -0,0 +1,590 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = pw_len * 2 * 8; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + w0[0] = a; + w0[1] = b; + w0[2] = c; + w0[3] = d; + w1[0] = salt_buf00; + w1[1] = salt_buf01; + w1[2] = salt_buf02; + w1[3] = salt_buf03; + w2[0] = salt_buf04; + w2[1] = salt_buf05; + w2[2] = salt_buf06; + w2[3] = salt_buf07; + w3[0] = salt_buf08; + w3[1] = salt_buf09; + w3[2] = salt_buf10; + w3[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + w3[2] = pw_len * 2 * 8; + w3[3] = 0; + + /** + * md4 + */ + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + w0[0] = a; + w0[1] = b; + w0[2] = c; + w0[3] = d; + w1[0] = salt_buf00; + w1[1] = salt_buf01; + w1[2] = salt_buf02; + w1[3] = salt_buf03; + w2[0] = salt_buf04; + w2[1] = salt_buf05; + w2[2] = salt_buf06; + w2[3] = salt_buf07; + w3[0] = salt_buf08; + w3[1] = salt_buf09; + w3[2] = salt_buf10; + w3[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01100_a1-pure.cl b/OpenCL/m01100_a1-pure.cl deleted file mode 100644 index 56fa3bfda..000000000 --- a/OpenCL/m01100_a1-pure.cl +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md4.cl" - -__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx1 = ctx0; - - md4_update_global_utf16le (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx1); - - md4_ctx_t ctx; - - md4_init (&ctx); - - ctx.w0[0] = ctx1.h[0]; - ctx.w0[1] = ctx1.h[1]; - ctx.w0[2] = ctx1.h[2]; - ctx.w0[3] = ctx1.h[3]; - - ctx.len = 16; - - md4_update_utf16le (&ctx, s, salt_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md4_ctx_t ctx0; - - md4_init (&ctx0); - - md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md4_ctx_t ctx1 = ctx0; - - md4_update_global_utf16le (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md4_final (&ctx1); - - md4_ctx_t ctx; - - md4_init (&ctx); - - ctx.w0[0] = ctx1.h[0]; - ctx.w0[1] = ctx1.h[1]; - ctx.w0[2] = ctx1.h[2]; - ctx.w0[3] = ctx1.h[3]; - - ctx.len = 16; - - md4_update_utf16le (&ctx, s, salt_len); - - md4_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01100_a1.cl b/OpenCL/m01100_a1.cl index bbb36149f..56fa3bfda 100644 --- a/OpenCL/m01100_a1.cl +++ b/OpenCL/m01100_a1.cl @@ -3,348 +3,97 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md4.cl" -__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { + /** + * modifier + */ + + const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + /** * base */ - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); + const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 pw_buf0[4]; - u32 pw_buf1[4]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + u32x s[64] = { 0 }; - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) + for (int idx = 0; idx < salt_lenv; idx++) { - s_salt_buf[0] = salt_bufs[salt_pos]; + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + barrier (CLK_GLOBAL_MEM_FENCE); } - barrier (CLK_LOCAL_MEM_FENCE); + md4_ctx_t ctx0; - if (gid >= gid_max) return; + md4_init (&ctx0); - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] + md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global_utf16le (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md4_ctx_t ctx; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md4_init (&ctx); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + ctx.w0[0] = ctx1.h[0]; + ctx.w0[1] = ctx1.h[1]; + ctx.w0[2] = ctx1.h[2]; + ctx.w0[3] = ctx1.h[3]; - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.len = 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md4_update_utf16le (&ctx, s, salt_len); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md4_final (&ctx); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - w3[2] = pw_len * 2 * 8; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; - - w0[0] = a; - w0[1] = b; - w0[2] = c; - w0[3] = d; - w1[0] = salt_buf00; - w1[1] = salt_buf01; - w1[2] = salt_buf02; - w1[3] = salt_buf03; - w2[0] = salt_buf04; - w2[1] = salt_buf05; - w2[2] = salt_buf06; - w2[3] = salt_buf07; - w3[0] = salt_buf08; - w3[1] = salt_buf09; - w3[2] = salt_buf10; - w3[3] = 0; - - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** - * base + * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); + const u32 gid = get_global_id (0); if (gid >= gid_max) return; - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] - /** * digest */ @@ -357,234 +106,61 @@ __kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md4_ctx_t ctx0; + + md4_init (&ctx0); + + md4_update_global_utf16le (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md4_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md4_update_global_utf16le (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md4_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md4_ctx_t ctx; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md4_init (&ctx); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + ctx.w0[0] = ctx1.h[0]; + ctx.w0[1] = ctx1.h[1]; + ctx.w0[2] = ctx1.h[2]; + ctx.w0[3] = ctx1.h[3]; - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.len = 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md4_update_utf16le (&ctx, s, salt_len); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md4_final (&ctx); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - w3[2] = pw_len * 2 * 8; - w3[3] = 0; - - /** - * md4 - */ - - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; - - w0[0] = a; - w0[1] = b; - w0[2] = c; - w0[3] = d; - w1[0] = salt_buf00; - w1[1] = salt_buf01; - w1[2] = salt_buf02; - w1[3] = salt_buf03; - w2[0] = salt_buf04; - w2[1] = salt_buf05; - w2[2] = salt_buf06; - w2[3] = salt_buf07; - w3[0] = salt_buf08; - w3[1] = salt_buf09; - w3[2] = salt_buf10; - w3[3] = 0; - - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; - - MD4_STEP (MD4_Fo, a, b, c, d, w0[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3[3], MD4C00, MD4S03); - - MD4_STEP (MD4_Go, a, b, c, d, w0[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3[3], MD4C01, MD4S13); - - MD4_STEP (MD4_H , a, b, c, d, w0[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0[3], MD4C02, MD4S20); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD4_STEP (MD4_H , d, a, b, c, w2[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01100_a3-optimized.cl b/OpenCL/m01100_a3-optimized.cl new file mode 100644 index 000000000..86cfcebff --- /dev/null +++ b/OpenCL/m01100_a3-optimized.cl @@ -0,0 +1,722 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m01100m (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = a; + w0_t[1] = b; + w0_t[2] = c; + w0_t[3] = d; + w1_t[0] = salt_buf00; + w1_t[1] = salt_buf01; + w1_t[2] = salt_buf02; + w1_t[3] = salt_buf03; + w2_t[0] = salt_buf04; + w2_t[1] = salt_buf05; + w2_t[2] = salt_buf06; + w2_t[3] = salt_buf07; + w3_t[0] = salt_buf08; + w3_t[1] = salt_buf09; + w3_t[2] = salt_buf10; + w3_t[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3_t[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0_t[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0_t[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[3], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[3], MD4C02, MD4S23); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m01100s (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + #define salt_buf00 s_salt_buf[0].salt_buf[ 0] + #define salt_buf01 s_salt_buf[0].salt_buf[ 1] + #define salt_buf02 s_salt_buf[0].salt_buf[ 2] + #define salt_buf03 s_salt_buf[0].salt_buf[ 3] + #define salt_buf04 s_salt_buf[0].salt_buf[ 4] + #define salt_buf05 s_salt_buf[0].salt_buf[ 5] + #define salt_buf06 s_salt_buf[0].salt_buf[ 6] + #define salt_buf07 s_salt_buf[0].salt_buf[ 7] + #define salt_buf08 s_salt_buf[0].salt_buf[ 8] + #define salt_buf09 s_salt_buf[0].salt_buf[ 9] + #define salt_buf10 s_salt_buf[0].salt_buf[10] + + /** + * base + */ + + const u32 F_w0c00 = 0 + MD4C00; + const u32 F_w1c00 = w[ 1] + MD4C00; + const u32 F_w2c00 = w[ 2] + MD4C00; + const u32 F_w3c00 = w[ 3] + MD4C00; + const u32 F_w4c00 = w[ 4] + MD4C00; + const u32 F_w5c00 = w[ 5] + MD4C00; + const u32 F_w6c00 = w[ 6] + MD4C00; + const u32 F_w7c00 = w[ 7] + MD4C00; + const u32 F_w8c00 = w[ 8] + MD4C00; + const u32 F_w9c00 = w[ 9] + MD4C00; + const u32 F_wac00 = w[10] + MD4C00; + const u32 F_wbc00 = w[11] + MD4C00; + const u32 F_wcc00 = w[12] + MD4C00; + const u32 F_wdc00 = w[13] + MD4C00; + const u32 F_wec00 = w[14] + MD4C00; + const u32 F_wfc00 = w[15] + MD4C00; + + const u32 G_w0c01 = 0 + MD4C01; + const u32 G_w4c01 = w[ 4] + MD4C01; + const u32 G_w8c01 = w[ 8] + MD4C01; + const u32 G_wcc01 = w[12] + MD4C01; + const u32 G_w1c01 = w[ 1] + MD4C01; + const u32 G_w5c01 = w[ 5] + MD4C01; + const u32 G_w9c01 = w[ 9] + MD4C01; + const u32 G_wdc01 = w[13] + MD4C01; + const u32 G_w2c01 = w[ 2] + MD4C01; + const u32 G_w6c01 = w[ 6] + MD4C01; + const u32 G_wac01 = w[10] + MD4C01; + const u32 G_wec01 = w[14] + MD4C01; + const u32 G_w3c01 = w[ 3] + MD4C01; + const u32 G_w7c01 = w[ 7] + MD4C01; + const u32 G_wbc01 = w[11] + MD4C01; + const u32 G_wfc01 = w[15] + MD4C01; + + const u32 H_w0c02 = 0 + MD4C02; + const u32 H_w8c02 = w[ 8] + MD4C02; + const u32 H_w4c02 = w[ 4] + MD4C02; + const u32 H_wcc02 = w[12] + MD4C02; + const u32 H_w2c02 = w[ 2] + MD4C02; + const u32 H_wac02 = w[10] + MD4C02; + const u32 H_w6c02 = w[ 6] + MD4C02; + const u32 H_wec02 = w[14] + MD4C02; + const u32 H_w1c02 = w[ 1] + MD4C02; + const u32 H_w9c02 = w[ 9] + MD4C02; + const u32 H_w5c02 = w[ 5] + MD4C02; + const u32 H_wdc02 = w[13] + MD4C02; + const u32 H_w3c02 = w[ 3] + MD4C02; + const u32 H_wbc02 = w[11] + MD4C02; + const u32 H_w7c02 = w[ 7] + MD4C02; + const u32 H_wfc02 = w[15] + MD4C02; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x a = MD4M_A; + u32x b = MD4M_B; + u32x c = MD4M_C; + u32x d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); + MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); + MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); + MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); + MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); + MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); + MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); + MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); + MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); + MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); + MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); + MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); + MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + + a += MD4M_A; + b += MD4M_B; + c += MD4M_C; + d += MD4M_D; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = a; + w0_t[1] = b; + w0_t[2] = c; + w0_t[3] = d; + w1_t[0] = salt_buf00; + w1_t[1] = salt_buf01; + w1_t[2] = salt_buf02; + w1_t[3] = salt_buf03; + w2_t[0] = salt_buf04; + w2_t[1] = salt_buf05; + w2_t[2] = salt_buf06; + w2_t[3] = salt_buf07; + w3_t[0] = salt_buf08; + w3_t[1] = salt_buf09; + w3_t[2] = salt_buf10; + w3_t[3] = 0; + + a = MD4M_A; + b = MD4M_B; + c = MD4M_C; + d = MD4M_D; + + MD4_STEP (MD4_Fo, a, b, c, d, w0_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w0_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w0_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w0_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w1_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w1_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w1_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w1_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w2_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w2_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w2_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w2_t[3], MD4C00, MD4S03); + MD4_STEP (MD4_Fo, a, b, c, d, w3_t[0], MD4C00, MD4S00); + MD4_STEP (MD4_Fo, d, a, b, c, w3_t[1], MD4C00, MD4S01); + MD4_STEP (MD4_Fo, c, d, a, b, w3_t[2], MD4C00, MD4S02); + MD4_STEP (MD4_Fo, b, c, d, a, w3_t[3], MD4C00, MD4S03); + + MD4_STEP (MD4_Go, a, b, c, d, w0_t[0], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[0], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[0], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[0], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[1], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[1], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[1], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[1], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[2], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[2], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[2], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[2], MD4C01, MD4S13); + MD4_STEP (MD4_Go, a, b, c, d, w0_t[3], MD4C01, MD4S10); + MD4_STEP (MD4_Go, d, a, b, c, w1_t[3], MD4C01, MD4S11); + MD4_STEP (MD4_Go, c, d, a, b, w2_t[3], MD4C01, MD4S12); + MD4_STEP (MD4_Go, b, c, d, a, w3_t[3], MD4C01, MD4S13); + + MD4_STEP (MD4_H , a, b, c, d, w0_t[0], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[0], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[0], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[0], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[2], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[2], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[2], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[2], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[1], MD4C02, MD4S20); + MD4_STEP (MD4_H , d, a, b, c, w2_t[1], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[1], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[1], MD4C02, MD4S23); + MD4_STEP (MD4_H , a, b, c, d, w0_t[3], MD4C02, MD4S20); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD4_STEP (MD4_H , d, a, b, c, w2_t[3], MD4C02, MD4S21); + MD4_STEP (MD4_H , c, d, a, b, w1_t[3], MD4C02, MD4S22); + MD4_STEP (MD4_H , b, c, d, a, w3_t[3], MD4C02, MD4S23); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m01100m (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m01100m (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m01100s (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = pws[gid].i[14]; + w[15] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + __local salt_t s_salt_buf[1]; + + if (lid == 0) + { + s_salt_buf[0] = salt_bufs[salt_pos]; + + s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m01100s (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01100_a3-pure.cl b/OpenCL/m01100_a3-pure.cl deleted file mode 100644 index 461b834a8..000000000 --- a/OpenCL/m01100_a3-pure.cl +++ /dev/null @@ -1,200 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md4.cl" - -__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx0; - - md4_init_vector (&ctx0); - - md4_update_vector_utf16le (&ctx0, w, pw_len); - - md4_final_vector (&ctx0); - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - - ctx.len = 16; - - md4_update_vector_utf16le (&ctx, s, salt_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md4_ctx_vector_t ctx0; - - md4_init_vector (&ctx0); - - md4_update_vector_utf16le (&ctx0, w, pw_len); - - md4_final_vector (&ctx0); - - md4_ctx_vector_t ctx; - - md4_init_vector (&ctx); - - ctx.w0[0] = ctx0.h[0]; - ctx.w0[1] = ctx0.h[1]; - ctx.w0[2] = ctx0.h[2]; - ctx.w0[3] = ctx0.h[3]; - - ctx.len = 16; - - md4_update_vector_utf16le (&ctx, s, salt_len); - - md4_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01100_a3.cl b/OpenCL/m01100_a3.cl index 86cfcebff..461b834a8 100644 --- a/OpenCL/m01100_a3.cl +++ b/OpenCL/m01100_a3.cl @@ -11,92 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md4.cl" -void m01100m (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] + if (gid >= gid_max) return; /** * base */ - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; + const u32 pw_len = pws[gid].pw_len; - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -104,228 +66,50 @@ void m01100m (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global const u32x w0 = w0l | w0r; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + w[0] = w0; - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_ctx_vector_t ctx0; - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + md4_init_vector (&ctx0); - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + md4_update_vector_utf16le (&ctx0, w, pw_len); - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; + md4_final_vector (&ctx0); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md4_ctx_vector_t ctx; - w0_t[0] = a; - w0_t[1] = b; - w0_t[2] = c; - w0_t[3] = d; - w1_t[0] = salt_buf00; - w1_t[1] = salt_buf01; - w1_t[2] = salt_buf02; - w1_t[3] = salt_buf03; - w2_t[0] = salt_buf04; - w2_t[1] = salt_buf05; - w2_t[2] = salt_buf06; - w2_t[3] = salt_buf07; - w3_t[0] = salt_buf08; - w3_t[1] = salt_buf09; - w3_t[2] = salt_buf10; - w3_t[3] = 0; + md4_init_vector (&ctx); - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3_t[3], MD4C00, MD4S03); + ctx.len = 16; - MD4_STEP (MD4_Go, a, b, c, d, w0_t[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[3], MD4C01, MD4S13); + md4_update_vector_utf16le (&ctx, s, salt_len); - MD4_STEP (MD4_H , a, b, c, d, w0_t[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[3], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[3], MD4C02, MD4S23); + md4_final_vector (&ctx); - COMPARE_M_SIMD (a, d, c, b); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01100s (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - #define salt_buf00 s_salt_buf[0].salt_buf[ 0] - #define salt_buf01 s_salt_buf[0].salt_buf[ 1] - #define salt_buf02 s_salt_buf[0].salt_buf[ 2] - #define salt_buf03 s_salt_buf[0].salt_buf[ 3] - #define salt_buf04 s_salt_buf[0].salt_buf[ 4] - #define salt_buf05 s_salt_buf[0].salt_buf[ 5] - #define salt_buf06 s_salt_buf[0].salt_buf[ 6] - #define salt_buf07 s_salt_buf[0].salt_buf[ 7] - #define salt_buf08 s_salt_buf[0].salt_buf[ 8] - #define salt_buf09 s_salt_buf[0].salt_buf[ 9] - #define salt_buf10 s_salt_buf[0].salt_buf[10] - - /** - * base - */ - - const u32 F_w0c00 = 0 + MD4C00; - const u32 F_w1c00 = w[ 1] + MD4C00; - const u32 F_w2c00 = w[ 2] + MD4C00; - const u32 F_w3c00 = w[ 3] + MD4C00; - const u32 F_w4c00 = w[ 4] + MD4C00; - const u32 F_w5c00 = w[ 5] + MD4C00; - const u32 F_w6c00 = w[ 6] + MD4C00; - const u32 F_w7c00 = w[ 7] + MD4C00; - const u32 F_w8c00 = w[ 8] + MD4C00; - const u32 F_w9c00 = w[ 9] + MD4C00; - const u32 F_wac00 = w[10] + MD4C00; - const u32 F_wbc00 = w[11] + MD4C00; - const u32 F_wcc00 = w[12] + MD4C00; - const u32 F_wdc00 = w[13] + MD4C00; - const u32 F_wec00 = w[14] + MD4C00; - const u32 F_wfc00 = w[15] + MD4C00; - - const u32 G_w0c01 = 0 + MD4C01; - const u32 G_w4c01 = w[ 4] + MD4C01; - const u32 G_w8c01 = w[ 8] + MD4C01; - const u32 G_wcc01 = w[12] + MD4C01; - const u32 G_w1c01 = w[ 1] + MD4C01; - const u32 G_w5c01 = w[ 5] + MD4C01; - const u32 G_w9c01 = w[ 9] + MD4C01; - const u32 G_wdc01 = w[13] + MD4C01; - const u32 G_w2c01 = w[ 2] + MD4C01; - const u32 G_w6c01 = w[ 6] + MD4C01; - const u32 G_wac01 = w[10] + MD4C01; - const u32 G_wec01 = w[14] + MD4C01; - const u32 G_w3c01 = w[ 3] + MD4C01; - const u32 G_w7c01 = w[ 7] + MD4C01; - const u32 G_wbc01 = w[11] + MD4C01; - const u32 G_wfc01 = w[15] + MD4C01; - - const u32 H_w0c02 = 0 + MD4C02; - const u32 H_w8c02 = w[ 8] + MD4C02; - const u32 H_w4c02 = w[ 4] + MD4C02; - const u32 H_wcc02 = w[12] + MD4C02; - const u32 H_w2c02 = w[ 2] + MD4C02; - const u32 H_wac02 = w[10] + MD4C02; - const u32 H_w6c02 = w[ 6] + MD4C02; - const u32 H_wec02 = w[14] + MD4C02; - const u32 H_w1c02 = w[ 1] + MD4C02; - const u32 H_w9c02 = w[ 9] + MD4C02; - const u32 H_w5c02 = w[ 5] + MD4C02; - const u32 H_wdc02 = w[13] + MD4C02; - const u32 H_w3c02 = w[ 3] + MD4C02; - const u32 H_wbc02 = w[11] + MD4C02; - const u32 H_w7c02 = w[ 7] + MD4C02; - const u32 H_wfc02 = w[15] + MD4C02; + if (gid >= gid_max) return; /** * digest @@ -339,11 +123,41 @@ void m01100s (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -351,372 +165,36 @@ void m01100s (__local salt_t *s_salt_buf, u32 w[16], const u32 pw_len, __global const u32x w0 = w0l | w0r; - u32x a = MD4M_A; - u32x b = MD4M_B; - u32x c = MD4M_C; - u32x d = MD4M_D; + w[0] = w0; - MD4_STEP (MD4_Fo, a, b, c, d, w0, F_w0c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w1c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w2c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w3c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w4c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w5c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_w6c00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_w7c00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_w8c00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_w9c00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wac00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wbc00, MD4S03); - MD4_STEP0(MD4_Fo, a, b, c, d, F_wcc00, MD4S00); - MD4_STEP0(MD4_Fo, d, a, b, c, F_wdc00, MD4S01); - MD4_STEP0(MD4_Fo, c, d, a, b, F_wec00, MD4S02); - MD4_STEP0(MD4_Fo, b, c, d, a, F_wfc00, MD4S03); + md4_ctx_vector_t ctx0; - MD4_STEP (MD4_Go, a, b, c, d, w0, G_w0c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w4c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w8c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wcc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w1c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w5c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_w9c01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wdc01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w2c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w6c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wac01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wec01, MD4S13); - MD4_STEP0(MD4_Go, a, b, c, d, G_w3c01, MD4S10); - MD4_STEP0(MD4_Go, d, a, b, c, G_w7c01, MD4S11); - MD4_STEP0(MD4_Go, c, d, a, b, G_wbc01, MD4S12); - MD4_STEP0(MD4_Go, b, c, d, a, G_wfc01, MD4S13); + md4_init_vector (&ctx0); - MD4_STEP (MD4_H , a, b, c, d, w0, H_w0c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w8c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w4c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wcc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w2c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wac02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w6c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wec02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w1c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_w9c02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w5c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wdc02, MD4S23); - MD4_STEP0(MD4_H , a, b, c, d, H_w3c02, MD4S20); - MD4_STEP0(MD4_H , d, a, b, c, H_wbc02, MD4S21); - MD4_STEP0(MD4_H , c, d, a, b, H_w7c02, MD4S22); - MD4_STEP0(MD4_H , b, c, d, a, H_wfc02, MD4S23); + md4_update_vector_utf16le (&ctx0, w, pw_len); - a += MD4M_A; - b += MD4M_B; - c += MD4M_C; - d += MD4M_D; + md4_final_vector (&ctx0); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md4_ctx_vector_t ctx; - w0_t[0] = a; - w0_t[1] = b; - w0_t[2] = c; - w0_t[3] = d; - w1_t[0] = salt_buf00; - w1_t[1] = salt_buf01; - w1_t[2] = salt_buf02; - w1_t[3] = salt_buf03; - w2_t[0] = salt_buf04; - w2_t[1] = salt_buf05; - w2_t[2] = salt_buf06; - w2_t[3] = salt_buf07; - w3_t[0] = salt_buf08; - w3_t[1] = salt_buf09; - w3_t[2] = salt_buf10; - w3_t[3] = 0; + md4_init_vector (&ctx); - a = MD4M_A; - b = MD4M_B; - c = MD4M_C; - d = MD4M_D; + ctx.w0[0] = ctx0.h[0]; + ctx.w0[1] = ctx0.h[1]; + ctx.w0[2] = ctx0.h[2]; + ctx.w0[3] = ctx0.h[3]; - MD4_STEP (MD4_Fo, a, b, c, d, w0_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w0_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w0_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w0_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w1_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w1_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w1_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w1_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w2_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w2_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w2_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w2_t[3], MD4C00, MD4S03); - MD4_STEP (MD4_Fo, a, b, c, d, w3_t[0], MD4C00, MD4S00); - MD4_STEP (MD4_Fo, d, a, b, c, w3_t[1], MD4C00, MD4S01); - MD4_STEP (MD4_Fo, c, d, a, b, w3_t[2], MD4C00, MD4S02); - MD4_STEP (MD4_Fo, b, c, d, a, w3_t[3], MD4C00, MD4S03); + ctx.len = 16; - MD4_STEP (MD4_Go, a, b, c, d, w0_t[0], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[0], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[0], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[0], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[1], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[1], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[1], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[1], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[2], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[2], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[2], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[2], MD4C01, MD4S13); - MD4_STEP (MD4_Go, a, b, c, d, w0_t[3], MD4C01, MD4S10); - MD4_STEP (MD4_Go, d, a, b, c, w1_t[3], MD4C01, MD4S11); - MD4_STEP (MD4_Go, c, d, a, b, w2_t[3], MD4C01, MD4S12); - MD4_STEP (MD4_Go, b, c, d, a, w3_t[3], MD4C01, MD4S13); + md4_update_vector_utf16le (&ctx, s, salt_len); - MD4_STEP (MD4_H , a, b, c, d, w0_t[0], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[0], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[0], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[0], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[2], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[2], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[2], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[2], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[1], MD4C02, MD4S20); - MD4_STEP (MD4_H , d, a, b, c, w2_t[1], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[1], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[1], MD4C02, MD4S23); - MD4_STEP (MD4_H , a, b, c, d, w0_t[3], MD4C02, MD4S20); + md4_final_vector (&ctx); - if (MATCHES_NONE_VS (a, search[0])) continue; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD4_STEP (MD4_H , d, a, b, c, w2_t[3], MD4C02, MD4S21); - MD4_STEP (MD4_H , c, d, a, b, w1_t[3], MD4C02, MD4S22); - MD4_STEP (MD4_H , b, c, d, a, w3_t[3], MD4C02, MD4S23); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m01100m (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m01100m (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m01100s (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = pws[gid].i[14]; - w[15] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - __local salt_t s_salt_buf[1]; - - if (lid == 0) - { - s_salt_buf[0] = salt_bufs[salt_pos]; - - s_salt_buf[0].salt_buf[10] = (16 + s_salt_buf[0].salt_len) * 8; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m01100s (s_salt_buf, w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01300_a0-optimized.cl b/OpenCL/m01300_a0-optimized.cl new file mode 100644 index 000000000..4104ce7da --- /dev/null +++ b/OpenCL/m01300_a0-optimized.cl @@ -0,0 +1,369 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = 0; \ +} + +__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha224 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, f, c, g); + } +} + +__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha224 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + + if (MATCHES_NONE_VS (g, d_rev)) continue; + + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, f, c, g); + } +} + +__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01300_a0-pure.cl b/OpenCL/m01300_a0-pure.cl deleted file mode 100644 index 7607c95f8..000000000 --- a/OpenCL/m01300_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha224.cl" - -__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha224_ctx_t ctx; - - sha224_init (&ctx); - - sha224_update_swap (&ctx, w, pw_len); - - sha224_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha224_ctx_t ctx; - - sha224_init (&ctx); - - sha224_update_swap (&ctx, w, pw_len); - - sha224_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01300_a0.cl b/OpenCL/m01300_a0.cl index 4104ce7da..7607c95f8 100644 --- a/OpenCL/m01300_a0.cl +++ b/OpenCL/m01300_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,206 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha224.cl" -#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = 0; \ -} - -__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha224_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha224_init (&ctx); - /** - * sha224 - */ + sha224_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha224_final (&ctx); - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, f, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -225,145 +92,43 @@ __kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + const u32 pw_len = pws[gid].pw_len; - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha224_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha224_init (&ctx); - /** - * sha224 - */ + sha224_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha224_final (&ctx); - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - - if (MATCHES_NONE_VS (g, d_rev)) continue; - - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, f, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01300_a1-optimized.cl b/OpenCL/m01300_a1-optimized.cl new file mode 100644 index 000000000..ed0ae0f59 --- /dev/null +++ b/OpenCL/m01300_a1-optimized.cl @@ -0,0 +1,483 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = 0; \ +} + +__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha224 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, f, c, g); + } +} + +__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha224 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + + if (MATCHES_NONE_VS (g, d_rev)) continue; + + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, f, c, g); + } +} + +__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01300_a1-pure.cl b/OpenCL/m01300_a1-pure.cl deleted file mode 100644 index 3f48fc9e2..000000000 --- a/OpenCL/m01300_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha224.cl" - -__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha224_ctx_t ctx0; - - sha224_init (&ctx0); - - sha224_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha224_ctx_t ctx = ctx0; - - sha224_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha224_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha224_ctx_t ctx0; - - sha224_init (&ctx0); - - sha224_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha224_ctx_t ctx = ctx0; - - sha224_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha224_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01300_a1.cl b/OpenCL/m01300_a1.cl index ed0ae0f59..3f48fc9e2 100644 --- a/OpenCL/m01300_a1.cl +++ b/OpenCL/m01300_a1.cl @@ -3,271 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha224.cl" -#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = 0; \ -} - -__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha224_ctx_t ctx0; - if (gid >= gid_max) return; + sha224_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha224_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha224_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha224_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha224_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha224 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, f, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -281,203 +79,32 @@ __kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + sha224_ctx_t ctx0; - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + sha224_init (&ctx0); + + sha224_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha224_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha224_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha224_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha224 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - - if (MATCHES_NONE_VS (g, d_rev)) continue; - - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, f, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01300_a3-optimized.cl b/OpenCL/m01300_a3-optimized.cl new file mode 100644 index 000000000..daf71318b --- /dev/null +++ b/OpenCL/m01300_a3-optimized.cl @@ -0,0 +1,523 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = 0; \ +} + +void m01300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, f, c, g); + } +} + +void m01300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA224M_A; + u32x b = SHA224M_B; + u32x c = SHA224M_C; + u32x d = SHA224M_D; + u32x e = SHA224M_E; + u32x f = SHA224M_F; + u32x g = SHA224M_G; + u32x h = SHA224M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + + if (MATCHES_NONE_VS (g, d_rev)) continue; + + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, f, c, g); + } +} + +__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01300_a3-pure.cl b/OpenCL/m01300_a3-pure.cl deleted file mode 100644 index cbb1203a8..000000000 --- a/OpenCL/m01300_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha224.cl" - -__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha224_ctx_vector_t ctx; - - sha224_init_vector (&ctx); - - sha224_update_vector (&ctx, w, pw_len); - - sha224_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha224_ctx_vector_t ctx; - - sha224_init_vector (&ctx); - - sha224_update_vector (&ctx, w, pw_len); - - sha224_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01300_a3.cl b/OpenCL/m01300_a3.cl index daf71318b..cbb1203a8 100644 --- a/OpenCL/m01300_a3.cl +++ b/OpenCL/m01300_a3.cl @@ -11,34 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha224.cl" -#define SHA224_STEP_REV(a,b,c,d,e,f,g) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = 0; \ -} - -void m01300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01300_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -46,112 +53,35 @@ void m01300m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; + sha224_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha224_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha224_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha224_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - COMPARE_M_SIMD (d, f, c, g); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01300_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -166,26 +96,27 @@ void m01300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + const u32 pw_len = pws[gid].pw_len; - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); - SHA224_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -193,331 +124,21 @@ void m01300s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA224M_A; - u32x b = SHA224M_B; - u32x c = SHA224M_C; - u32x d = SHA224M_D; - u32x e = SHA224M_E; - u32x f = SHA224M_F; - u32x g = SHA224M_G; - u32x h = SHA224M_H; + sha224_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha224_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha224_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha224_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - if (MATCHES_NONE_VS (g, d_rev)) continue; - - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, f, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01300_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01300_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01300_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01300_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01300_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01300_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01300s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01400_a0-optimized.cl b/OpenCL/m01400_a0-optimized.cl new file mode 100644 index 000000000..940fa7a8e --- /dev/null +++ b/OpenCL/m01400_a0-optimized.cl @@ -0,0 +1,372 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01400_a0-pure.cl b/OpenCL/m01400_a0-pure.cl deleted file mode 100644 index 3f07773af..000000000 --- a/OpenCL/m01400_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01400_a0.cl b/OpenCL/m01400_a0.cl index 940fa7a8e..3f07773af 100644 --- a/OpenCL/m01400_a0.cl +++ b/OpenCL/m01400_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,207 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha256_init (&ctx); - /** - * sha256 - */ + sha256_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha256_final (&ctx); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -226,147 +92,43 @@ __kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha256_init (&ctx); - /** - * sha256 - */ + sha256_update_swap (&ctx, w, pw_len); - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_len * 8; + sha256_final (&ctx); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01400_a1-optimized.cl b/OpenCL/m01400_a1-optimized.cl new file mode 100644 index 000000000..f47c7bbd3 --- /dev/null +++ b/OpenCL/m01400_a1-optimized.cl @@ -0,0 +1,486 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01400_a1-pure.cl b/OpenCL/m01400_a1-pure.cl deleted file mode 100644 index 5ab9c109b..000000000 --- a/OpenCL/m01400_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01400_a1.cl b/OpenCL/m01400_a1.cl index f47c7bbd3..5ab9c109b 100644 --- a/OpenCL/m01400_a1.cl +++ b/OpenCL/m01400_a1.cl @@ -3,272 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha256_ctx_t ctx0; - if (gid >= gid_max) return; + sha256_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -282,205 +79,32 @@ __kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + sha256_ctx_t ctx0; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01400_a3-optimized.cl b/OpenCL/m01400_a3-optimized.cl new file mode 100644 index 000000000..92d76d6fa --- /dev/null +++ b/OpenCL/m01400_a3-optimized.cl @@ -0,0 +1,526 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +void m01400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +void m01400s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01400_a3-pure.cl b/OpenCL/m01400_a3-pure.cl deleted file mode 100644 index 6595c7b9a..000000000 --- a/OpenCL/m01400_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01400_a3.cl b/OpenCL/m01400_a3.cl index 92d76d6fa..6595c7b9a 100644 --- a/OpenCL/m01400_a3.cl +++ b/OpenCL/m01400_a3.cl @@ -11,35 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -void m01400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01400_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -47,112 +53,35 @@ void m01400m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01400s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01400_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -167,28 +96,27 @@ void m01400s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -196,331 +124,21 @@ void m01400s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01400_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01400_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01400_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01400_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01400_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01400_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01400s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01410_a0-optimized.cl b/OpenCL/m01410_a0-optimized.cl new file mode 100644 index 000000000..c0c8ee435 --- /dev/null +++ b/OpenCL/m01410_a0-optimized.cl @@ -0,0 +1,518 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01410_a0-pure.cl b/OpenCL/m01410_a0-pure.cl deleted file mode 100644 index 185c9e132..000000000 --- a/OpenCL/m01410_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_swap (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_swap (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01410_a0.cl b/OpenCL/m01410_a0.cl index c0c8ee435..185c9e132 100644 --- a/OpenCL/m01410_a0.cl +++ b/OpenCL/m01410_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,308 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - /** - * append salt - */ + sha256_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha256_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha256_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha256_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -327,192 +107,58 @@ __kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - /** - * append salt - */ + sha256_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha256_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha256_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha256_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01410_a1-optimized.cl b/OpenCL/m01410_a1-optimized.cl new file mode 100644 index 000000000..e55813078 --- /dev/null +++ b/OpenCL/m01410_a1-optimized.cl @@ -0,0 +1,636 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01410_a1-pure.cl b/OpenCL/m01410_a1-pure.cl deleted file mode 100644 index bf7a01885..000000000 --- a/OpenCL/m01410_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01410_a1.cl b/OpenCL/m01410_a1.cl index e55813078..bf7a01885 100644 --- a/OpenCL/m01410_a1.cl +++ b/OpenCL/m01410_a1.cl @@ -3,375 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha256_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -385,252 +94,47 @@ __kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha256_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01410_a3-optimized.cl b/OpenCL/m01410_a3-optimized.cl new file mode 100644 index 000000000..f2ae3de7c --- /dev/null +++ b/OpenCL/m01410_a3-optimized.cl @@ -0,0 +1,577 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +void m01410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +void m01410s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01410_a3-pure.cl b/OpenCL/m01410_a3-pure.cl deleted file mode 100644 index a1a7bd150..000000000 --- a/OpenCL/m01410_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_update_vector (&ctx, s, salt_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_update_vector (&ctx, s, salt_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01410_a3.cl b/OpenCL/m01410_a3.cl index f2ae3de7c..a1a7bd150 100644 --- a/OpenCL/m01410_a3.cl +++ b/OpenCL/m01410_a3.cl @@ -11,86 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -void m01410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01410_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + u32x w[64] = { 0 }; - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); - w[15] = pw_salt_len * 8; + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -98,112 +66,37 @@ void m01410m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_update_vector (&ctx, s, salt_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + sha256_final_vector (&ctx); - COMPARE_M_SIMD (d, h, c, g); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01410s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01410_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -218,28 +111,40 @@ void m01410s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -247,331 +152,23 @@ void m01410s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_update_vector (&ctx, s, salt_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + sha256_final_vector (&ctx); - if (MATCHES_NONE_VS (h, d_rev)) continue; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01410_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01410_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01410_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01410_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01410_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01410_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01410s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01420_a0-optimized.cl b/OpenCL/m01420_a0-optimized.cl new file mode 100644 index 000000000..868e263cb --- /dev/null +++ b/OpenCL/m01420_a0-optimized.cl @@ -0,0 +1,478 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01420_a0-pure.cl b/OpenCL/m01420_a0-pure.cl deleted file mode 100644 index 4733245f6..000000000 --- a/OpenCL/m01420_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx = ctx0; - - sha256_update_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx = ctx0; - - sha256_update_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01420_a0.cl b/OpenCL/m01420_a0.cl index 868e263cb..4733245f6 100644 --- a/OpenCL/m01420_a0.cl +++ b/OpenCL/m01420_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,288 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx = ctx0; - /** - * prepend salt - */ + sha256_update_swap (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + sha256_final (&ctx); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -307,172 +96,47 @@ __kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx = ctx0; - /** - * prepend salt - */ + sha256_update_swap (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + sha256_final (&ctx); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01420_a1-optimized.cl b/OpenCL/m01420_a1-optimized.cl new file mode 100644 index 000000000..dfa57ed32 --- /dev/null +++ b/OpenCL/m01420_a1-optimized.cl @@ -0,0 +1,592 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01420_a1-pure.cl b/OpenCL/m01420_a1-pure.cl deleted file mode 100644 index 482c8f72f..000000000 --- a/OpenCL/m01420_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01420_a1.cl b/OpenCL/m01420_a1.cl index dfa57ed32..482c8f72f 100644 --- a/OpenCL/m01420_a1.cl +++ b/OpenCL/m01420_a1.cl @@ -3,353 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha256_ctx_t ctx0; - if (gid >= gid_max) return; + sha256_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -363,230 +81,34 @@ __kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + sha256_ctx_t ctx0; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha256_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01420_a3-optimized.cl b/OpenCL/m01420_a3-optimized.cl new file mode 100644 index 000000000..21cb7c8cb --- /dev/null +++ b/OpenCL/m01420_a3-optimized.cl @@ -0,0 +1,742 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +void m01420m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha256 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +void m01420s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha256 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01420_a3-pure.cl b/OpenCL/m01420_a3-pure.cl deleted file mode 100644 index 02784ecd0..000000000 --- a/OpenCL/m01420_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector_from_scalar (&ctx, &ctx0); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector_from_scalar (&ctx, &ctx0); - - sha256_update_vector (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01420_a3.cl b/OpenCL/m01420_a3.cl index 21cb7c8cb..02784ecd0 100644 --- a/OpenCL/m01420_a3.cl +++ b/OpenCL/m01420_a3.cl @@ -11,229 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -void m01420m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01420_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha256_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha256_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha256 - */ + sha256_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha256_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_update_vector (&ctx, w, pw_len); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01420s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01420_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -248,495 +102,55 @@ void m01420s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha256_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha256_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; - - /** - * reverse - */ - - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; - - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha256 - */ + sha256_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha256_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_update_vector (&ctx, w, pw_len); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01420_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01420_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01420_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01420_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01420_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01420_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01420s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01430_a0-optimized.cl b/OpenCL/m01430_a0-optimized.cl new file mode 100644 index 000000000..0cb9e074e --- /dev/null +++ b/OpenCL/m01430_a0-optimized.cl @@ -0,0 +1,528 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01430_a0-pure.cl b/OpenCL/m01430_a0-pure.cl deleted file mode 100644 index 45ef46ba9..000000000 --- a/OpenCL/m01430_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_utf16le_swap (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_utf16le_swap (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01430_a0.cl b/OpenCL/m01430_a0.cl index 0cb9e074e..45ef46ba9 100644 --- a/OpenCL/m01430_a0.cl +++ b/OpenCL/m01430_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,313 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha256_init (&ctx); - const u32x out_len2 = out_len * 2; + sha256_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha256_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha256_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -332,197 +107,58 @@ __kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha256_init (&ctx); - const u32x out_len2 = out_len * 2; + sha256_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha256_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha256_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01430_a1-optimized.cl b/OpenCL/m01430_a1-optimized.cl new file mode 100644 index 000000000..ccbede77b --- /dev/null +++ b/OpenCL/m01430_a1-optimized.cl @@ -0,0 +1,646 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01430_a1-pure.cl b/OpenCL/m01430_a1-pure.cl deleted file mode 100644 index 2d0efa682..000000000 --- a/OpenCL/m01430_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01430_a1.cl b/OpenCL/m01430_a1.cl index ccbede77b..2d0efa682 100644 --- a/OpenCL/m01430_a1.cl +++ b/OpenCL/m01430_a1.cl @@ -3,380 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha256_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -390,257 +94,47 @@ __kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha256_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01430_a3-optimized.cl b/OpenCL/m01430_a3-optimized.cl new file mode 100644 index 000000000..8d60faddc --- /dev/null +++ b/OpenCL/m01430_a3-optimized.cl @@ -0,0 +1,577 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +void m01430m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +void m01430s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t = w0; + u32x w1_t = w[ 1]; + u32x w2_t = w[ 2]; + u32x w3_t = w[ 3]; + u32x w4_t = w[ 4]; + u32x w5_t = w[ 5]; + u32x w6_t = w[ 6]; + u32x w7_t = w[ 7]; + u32x w8_t = w[ 8]; + u32x w9_t = w[ 9]; + u32x wa_t = w[10]; + u32x wb_t = w[11]; + u32x wc_t = w[12]; + u32x wd_t = w[13]; + u32x we_t = w[14]; + u32x wf_t = w[15]; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01430_a3-pure.cl b/OpenCL/m01430_a3-pure.cl deleted file mode 100644 index 2a6ccc3f8..000000000 --- a/OpenCL/m01430_a3-pure.cl +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector_utf16beN (&ctx, w, pw_len); - - sha256_update_vector (&ctx, s, salt_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - - -__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector (&ctx); - - sha256_update_vector_utf16beN (&ctx, w, pw_len); - - sha256_update_vector (&ctx, s, salt_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01430_a3.cl b/OpenCL/m01430_a3.cl index 8d60faddc..2a6ccc3f8 100644 --- a/OpenCL/m01430_a3.cl +++ b/OpenCL/m01430_a3.cl @@ -11,86 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -void m01430m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01430_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + u32x w[64] = { 0 }; - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); - w[15] = pw_salt_len * 8; + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -98,112 +66,38 @@ void m01430m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector_utf16beN (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_update_vector (&ctx, s, salt_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + sha256_final_vector (&ctx); - COMPARE_M_SIMD (d, h, c, g); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01430s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) + +__kernel void m01430_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -218,28 +112,40 @@ void m01430s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -247,331 +153,23 @@ void m01430s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t = w0; - u32x w1_t = w[ 1]; - u32x w2_t = w[ 2]; - u32x w3_t = w[ 3]; - u32x w4_t = w[ 4]; - u32x w5_t = w[ 5]; - u32x w6_t = w[ 6]; - u32x w7_t = w[ 7]; - u32x w8_t = w[ 8]; - u32x w9_t = w[ 9]; - u32x wa_t = w[10]; - u32x wb_t = w[11]; - u32x wc_t = w[12]; - u32x wd_t = w[13]; - u32x we_t = w[14]; - u32x wf_t = w[15]; + w[0] = w0; - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_ctx_vector_t ctx; - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_init_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + sha256_update_vector_utf16beN (&ctx, w, pw_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + sha256_update_vector (&ctx, s, salt_len); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + sha256_final_vector (&ctx); - if (MATCHES_NONE_VS (h, d_rev)) continue; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01430_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01430_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01430_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01430_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01430_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01430_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01430s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01440_a0-optimized.cl b/OpenCL/m01440_a0-optimized.cl new file mode 100644 index 000000000..5aa26a5c0 --- /dev/null +++ b/OpenCL/m01440_a0-optimized.cl @@ -0,0 +1,488 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = out_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01440_a0-pure.cl b/OpenCL/m01440_a0-pure.cl deleted file mode 100644 index 4dcf235e0..000000000 --- a/OpenCL/m01440_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx = ctx0; - - sha256_update_utf16le_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_ctx_t ctx = ctx0; - - sha256_update_utf16le_swap (&ctx, w, pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01440_a0.cl b/OpenCL/m01440_a0.cl index 5aa26a5c0..4dcf235e0 100644 --- a/OpenCL/m01440_a0.cl +++ b/OpenCL/m01440_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,293 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha256_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha256_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -312,177 +96,47 @@ __kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + const u32 pw_len = pws[gid].pw_len; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_ctx_t ctx0; + + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha256_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha256_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = out_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01440_a1-optimized.cl b/OpenCL/m01440_a1-optimized.cl new file mode 100644 index 000000000..7bf1176b6 --- /dev/null +++ b/OpenCL/m01440_a1-optimized.cl @@ -0,0 +1,602 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha256 + */ + + u32x w0_t = swap32 (w0[0]); + u32x w1_t = swap32 (w0[1]); + u32x w2_t = swap32 (w0[2]); + u32x w3_t = swap32 (w0[3]); + u32x w4_t = swap32 (w1[0]); + u32x w5_t = swap32 (w1[1]); + u32x w6_t = swap32 (w1[2]); + u32x w7_t = swap32 (w1[3]); + u32x w8_t = swap32 (w2[0]); + u32x w9_t = swap32 (w2[1]); + u32x wa_t = swap32 (w2[2]); + u32x wb_t = swap32 (w2[3]); + u32x wc_t = swap32 (w3[0]); + u32x wd_t = swap32 (w3[1]); + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01440_a1-pure.cl b/OpenCL/m01440_a1-pure.cl deleted file mode 100644 index 269014b5b..000000000 --- a/OpenCL/m01440_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha256_ctx_t ctx = ctx0; - - sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha256_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01440_a1.cl b/OpenCL/m01440_a1.cl index 7bf1176b6..269014b5b 100644 --- a/OpenCL/m01440_a1.cl +++ b/OpenCL/m01440_a1.cl @@ -3,358 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha256_ctx_t ctx0; - if (gid >= gid_max) return; + sha256_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -368,235 +81,34 @@ __kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rule }; /** - * reverse + * base */ - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + sha256_ctx_t ctx0; - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + sha256_init (&ctx0); + + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha256_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha256_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha256_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha256_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha256 - */ - - u32x w0_t = swap32 (w0[0]); - u32x w1_t = swap32 (w0[1]); - u32x w2_t = swap32 (w0[2]); - u32x w3_t = swap32 (w0[3]); - u32x w4_t = swap32 (w1[0]); - u32x w5_t = swap32 (w1[1]); - u32x w6_t = swap32 (w1[2]); - u32x w7_t = swap32 (w1[3]); - u32x w8_t = swap32 (w2[0]); - u32x w9_t = swap32 (w2[1]); - u32x wa_t = swap32 (w2[2]); - u32x wb_t = swap32 (w2[3]); - u32x wc_t = swap32 (w3[0]); - u32x wd_t = swap32 (w3[1]); - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; - - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; - - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01440_a3-optimized.cl b/OpenCL/m01440_a3-optimized.cl new file mode 100644 index 000000000..f142dfdbc --- /dev/null +++ b/OpenCL/m01440_a3-optimized.cl @@ -0,0 +1,742 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ +{ \ + u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ + u32 t1 = a - t2; \ + a = b; \ + b = c; \ + c = d; \ + d = e - t1; \ + e = f; \ + f = g; \ + g = h; \ + h = 0; \ +} + +void m01440m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha256 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_M_SIMD (d, h, c, g); + } +} + +void m01440s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * reverse + */ + + u32 a_rev = digests_buf[digests_offset].digest_buf[0]; + u32 b_rev = digests_buf[digests_offset].digest_buf[1]; + u32 c_rev = digests_buf[digests_offset].digest_buf[2]; + u32 d_rev = digests_buf[digests_offset].digest_buf[3]; + u32 e_rev = digests_buf[digests_offset].digest_buf[4]; + u32 f_rev = digests_buf[digests_offset].digest_buf[5]; + u32 g_rev = digests_buf[digests_offset].digest_buf[6]; + u32 h_rev = digests_buf[digests_offset].digest_buf[7]; + + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha256 + */ + + u32x w0_t = t0[0]; + u32x w1_t = t0[1]; + u32x w2_t = t0[2]; + u32x w3_t = t0[3]; + u32x w4_t = t1[0]; + u32x w5_t = t1[1]; + u32x w6_t = t1[2]; + u32x w7_t = t1[3]; + u32x w8_t = t2[0]; + u32x w9_t = t2[1]; + u32x wa_t = t2[2]; + u32x wb_t = t2[3]; + u32x wc_t = t3[0]; + u32x wd_t = t3[1]; + u32x we_t = 0; + u32x wf_t = pw_salt_len * 8; + + u32x a = SHA256M_A; + u32x b = SHA256M_B; + u32x c = SHA256M_C; + u32x d = SHA256M_D; + u32x e = SHA256M_E; + u32x f = SHA256M_F; + u32x g = SHA256M_G; + u32x h = SHA256M_H; + + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); + + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); + + if (MATCHES_NONE_VS (h, d_rev)) continue; + + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); + + COMPARE_S_SIMD (d, h, c, g); + } +} + +__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01440_a3-pure.cl b/OpenCL/m01440_a3-pure.cl deleted file mode 100644 index a01604ac3..000000000 --- a/OpenCL/m01440_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector_from_scalar (&ctx, &ctx0); - - sha256_update_vector_utf16beN (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_ctx_t ctx0; - - sha256_init (&ctx0); - - sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_ctx_vector_t ctx; - - sha256_init_vector_from_scalar (&ctx, &ctx0); - - sha256_update_vector_utf16beN (&ctx, w, pw_len); - - sha256_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01440_a3.cl b/OpenCL/m01440_a3.cl index f142dfdbc..a01604ac3 100644 --- a/OpenCL/m01440_a3.cl +++ b/OpenCL/m01440_a3.cl @@ -11,229 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -#define SHA256_STEP_REV(a,b,c,d,e,f,g,h) \ -{ \ - u32 t2 = SHA256_S2_S(b) + SHA256_F0o(b,c,d); \ - u32 t1 = a - t2; \ - a = b; \ - b = c; \ - c = d; \ - d = e - t1; \ - e = f; \ - f = g; \ - g = h; \ - h = 0; \ -} - -void m01440m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01440_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha256_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha256_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha256 - */ + sha256_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha256_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_update_vector_utf16beN (&ctx, w, pw_len); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_M_SIMD (d, h, c, g); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01440s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01440_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -248,495 +102,55 @@ void m01440s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha256_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha256_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; - - /** - * reverse - */ - - u32 a_rev = digests_buf[digests_offset].digest_buf[0]; - u32 b_rev = digests_buf[digests_offset].digest_buf[1]; - u32 c_rev = digests_buf[digests_offset].digest_buf[2]; - u32 d_rev = digests_buf[digests_offset].digest_buf[3]; - u32 e_rev = digests_buf[digests_offset].digest_buf[4]; - u32 f_rev = digests_buf[digests_offset].digest_buf[5]; - u32 g_rev = digests_buf[digests_offset].digest_buf[6]; - u32 h_rev = digests_buf[digests_offset].digest_buf[7]; - - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); - SHA256_STEP_REV (a_rev, b_rev, c_rev, d_rev, e_rev, f_rev, g_rev, h_rev); + sha256_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha256 - */ + sha256_ctx_vector_t ctx; - u32x w0_t = t0[0]; - u32x w1_t = t0[1]; - u32x w2_t = t0[2]; - u32x w3_t = t0[3]; - u32x w4_t = t1[0]; - u32x w5_t = t1[1]; - u32x w6_t = t1[2]; - u32x w7_t = t1[3]; - u32x w8_t = t2[0]; - u32x w9_t = t2[1]; - u32x wa_t = t2[2]; - u32x wb_t = t2[3]; - u32x wc_t = t3[0]; - u32x wd_t = t3[1]; - u32x we_t = 0; - u32x wf_t = pw_salt_len * 8; + sha256_init_vector_from_scalar (&ctx, &ctx0); - u32x a = SHA256M_A; - u32x b = SHA256M_B; - u32x c = SHA256M_C; - u32x d = SHA256M_D; - u32x e = SHA256M_E; - u32x f = SHA256M_F; - u32x g = SHA256M_G; - u32x h = SHA256M_H; + sha256_update_vector_utf16beN (&ctx, w, pw_len); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C00); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C01); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C02); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C03); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C04); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C05); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C06); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C07); - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C08); - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C09); - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C0a); - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C0b); - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C0c); - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C0d); - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C0e); - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C0f); + sha256_final_vector (&ctx); - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C10); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C11); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C12); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C13); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C14); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C15); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C16); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C17); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C18); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C19); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C1a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C1b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C1c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C1d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C1e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C1f); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C20); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C21); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C22); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C23); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C24); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C25); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C26); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C27); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C28); - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C29); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C2a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C2b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C2c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C2d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C2e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C2f); - - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, SHA256C30); - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, SHA256C31); - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, SHA256C32); - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, SHA256C33); - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, SHA256C34); - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, SHA256C35); - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, SHA256C36); - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, SHA256C37); - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, SHA256C38); - - if (MATCHES_NONE_VS (h, d_rev)) continue; - - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, SHA256C39); - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, SHA256C3a); - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, SHA256C3b); - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, SHA256C3c); - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, SHA256C3d); - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, SHA256C3e); - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, SHA256C3f); - - COMPARE_S_SIMD (d, h, c, g); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01440_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01440_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01440_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01440_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01440_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01440_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01440s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01450_a0-optimized.cl b/OpenCL/m01450_a0-optimized.cl new file mode 100644 index 000000000..2dd380433 --- /dev/null +++ b/OpenCL/m01450_a0-optimized.cl @@ -0,0 +1,480 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01450_a0-pure.cl b/OpenCL/m01450_a0-pure.cl deleted file mode 100644 index 21fa01b0c..000000000 --- a/OpenCL/m01450_a0-pure.cl +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_hmac_ctx_t ctx; - - sha256_hmac_init_swap (&ctx, w, pw_len); - - sha256_hmac_update (&ctx, s, salt_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_hmac_ctx_t ctx; - - sha256_hmac_init_swap (&ctx, w, pw_len); - - sha256_hmac_update (&ctx, s, salt_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01450_a0.cl b/OpenCL/m01450_a0.cl index 2dd380433..21fa01b0c 100644 --- a/OpenCL/m01450_a0.cl +++ b/OpenCL/m01450_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,397 +12,86 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha256_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha256_hmac_update (&ctx, s, salt_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_final (&ctx); - hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; - - u32x digest[8]; - - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -415,66 +104,57 @@ __kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha256_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha256_hmac_update (&ctx, s, salt_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_final (&ctx); - hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; - - u32x digest[8]; - - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01450_a1-optimized.cl b/OpenCL/m01450_a1-optimized.cl new file mode 100644 index 000000000..a6378ba55 --- /dev/null +++ b/OpenCL/m01450_a1-optimized.cl @@ -0,0 +1,614 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01450_a1-pure.cl b/OpenCL/m01450_a1-pure.cl deleted file mode 100644 index c482c7626..000000000 --- a/OpenCL/m01450_a1-pure.cl +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha256_hmac_ctx_vector_t ctx; - - sha256_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha256_hmac_update_vector (&ctx, s, salt_len); - - sha256_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha256_hmac_ctx_vector_t ctx; - - sha256_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha256_hmac_update_vector (&ctx, s, salt_len); - - sha256_hmac_final_vector (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01450_a1.cl b/OpenCL/m01450_a1.cl index a6378ba55..c482c7626 100644 --- a/OpenCL/m01450_a1.cl +++ b/OpenCL/m01450_a1.cl @@ -3,472 +3,113 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha256_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha256_hmac_init_vector (&ctx, c, pw_len + comb_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_update_vector (&ctx, s, salt_len); - hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + sha256_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[8]; - - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -481,134 +122,77 @@ __kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha256_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha256_hmac_init_vector (&ctx, c, pw_len + comb_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_update_vector (&ctx, s, salt_len); - hmac_sha256_pad (w0, w1, w2, w3, ipad, opad); + sha256_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (64 + salt_len) * 8; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - u32x digest[8]; - - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01450_a3-optimized.cl b/OpenCL/m01450_a3-optimized.cl new file mode 100644 index 000000000..bc0f2d83a --- /dev/null +++ b/OpenCL/m01450_a3-optimized.cl @@ -0,0 +1,726 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +void m01450m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +void m01450s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + salt_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01450_a3-pure.cl b/OpenCL/m01450_a3-pure.cl deleted file mode 100644 index c55b53eae..000000000 --- a/OpenCL/m01450_a3-pure.cl +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_hmac_ctx_vector_t ctx; - - sha256_hmac_init_vector (&ctx, w, pw_len); - - sha256_hmac_update_vector (&ctx, s, salt_len); - - sha256_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_hmac_ctx_vector_t ctx; - - sha256_hmac_init_vector (&ctx, w, pw_len); - - sha256_hmac_update_vector (&ctx, s, salt_len); - - sha256_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01450_a3.cl b/OpenCL/m01450_a3.cl index bc0f2d83a..c55b53eae 100644 --- a/OpenCL/m01450_a3.cl +++ b/OpenCL/m01450_a3.cl @@ -11,357 +11,90 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -void m01450m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01450_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha256_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha256_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_update_vector (&ctx, s, salt_len); - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha256_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + salt_len) * 8; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[8]; - - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01450s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01450_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + if (gid >= gid_max) return; /** * digest @@ -375,352 +108,63 @@ void m01450s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha256_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha256_hmac_init_vector (&ctx, w, pw_len); - u32x ipad[8]; - u32x opad[8]; + sha256_hmac_update_vector (&ctx, s, salt_len); - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha256_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + salt_len) * 8; + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; - u32x digest[8]; - - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01450_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01450_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01450_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01450_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01450_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01450_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01450s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01460_a0-optimized.cl b/OpenCL/m01460_a0-optimized.cl new file mode 100644 index 000000000..4b10864c2 --- /dev/null +++ b/OpenCL/m01460_a0-optimized.cl @@ -0,0 +1,510 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (64 + out_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (64 + out_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01460_a0-pure.cl b/OpenCL/m01460_a0-pure.cl deleted file mode 100644 index 0b14e52e9..000000000 --- a/OpenCL/m01460_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_t ctx0; - - sha256_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_hmac_ctx_t ctx = ctx0; - - sha256_hmac_update_swap (&ctx, w, pw_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_t ctx0; - - sha256_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha256_hmac_ctx_t ctx = ctx0; - - sha256_hmac_update_swap (&ctx, w, pw_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01460_a0.cl b/OpenCL/m01460_a0.cl index 4b10864c2..0b14e52e9 100644 --- a/OpenCL/m01460_a0.cl +++ b/OpenCL/m01460_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,443 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[8]; - u32x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha256_hmac_ctx_t ctx0; + + sha256_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha256_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (64 + out_len) * 8; + sha256_hmac_final (&ctx); - u32x digest[8]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[8]; - u32x opad[8]; - - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -461,50 +106,59 @@ __kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_hmac_ctx_t ctx0; + + sha256_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha256_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha256_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (64 + out_len) * 8; + sha256_hmac_final (&ctx); - u32x digest[8]; + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01460_a1-optimized.cl b/OpenCL/m01460_a1-optimized.cl new file mode 100644 index 000000000..a9249e1a2 --- /dev/null +++ b/OpenCL/m01460_a1-optimized.cl @@ -0,0 +1,628 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (64 + pw_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (64 + pw_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01460_a1-pure.cl b/OpenCL/m01460_a1-pure.cl deleted file mode 100644 index d4b0fdb8b..000000000 --- a/OpenCL/m01460_a1-pure.cl +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_t ctx0; - - sha256_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha256_hmac_ctx_t ctx = ctx0; - - sha256_hmac_update (&ctx, c, pw_len + comb_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_t ctx0; - - sha256_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha256_hmac_ctx_t ctx = ctx0; - - sha256_hmac_update (&ctx, c, pw_len + comb_len); - - sha256_hmac_final (&ctx); - - const u32 r0 = ctx.opad.h[DGST_R0]; - const u32 r1 = ctx.opad.h[DGST_R1]; - const u32 r2 = ctx.opad.h[DGST_R2]; - const u32 r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01460_a1.cl b/OpenCL/m01460_a1.cl index a9249e1a2..d4b0fdb8b 100644 --- a/OpenCL/m01460_a1.cl +++ b/OpenCL/m01460_a1.cl @@ -3,510 +3,115 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[8]; - u32x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha256_hmac_ctx_t ctx0; + + sha256_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha256_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (64 + pw_len) * 8; + sha256_hmac_update (&ctx, c, pw_len + comb_len); - u32x digest[8]; + sha256_hmac_final (&ctx); - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[8]; - u32x opad[8]; - - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -519,110 +124,79 @@ __kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_hmac_ctx_t ctx0; + + sha256_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha256_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (64 + pw_len) * 8; + sha256_hmac_update (&ctx, c, pw_len + comb_len); - u32x digest[8]; + sha256_hmac_final (&ctx); - hmac_sha256_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = ctx.opad.h[DGST_R0]; + const u32 r1 = ctx.opad.h[DGST_R1]; + const u32 r2 = ctx.opad.h[DGST_R2]; + const u32 r3 = ctx.opad.h[DGST_R3]; - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01460_a3-optimized.cl b/OpenCL/m01460_a3-optimized.cl new file mode 100644 index 000000000..156b75bcc --- /dev/null +++ b/OpenCL/m01460_a3-optimized.cl @@ -0,0 +1,726 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) +{ + u32x a = digest[0]; + u32x b = digest[1]; + u32x c = digest[2]; + u32x d = digest[3]; + u32x e = digest[4]; + u32x f = digest[5]; + u32x g = digest[6]; + u32x h = digest[7]; + + u32x w0_t = w0[0]; + u32x w1_t = w0[1]; + u32x w2_t = w0[2]; + u32x w3_t = w0[3]; + u32x w4_t = w1[0]; + u32x w5_t = w1[1]; + u32x w6_t = w1[2]; + u32x w7_t = w1[3]; + u32x w8_t = w2[0]; + u32x w9_t = w2[1]; + u32x wa_t = w2[2]; + u32x wb_t = w2[3]; + u32x wc_t = w3[0]; + u32x wd_t = w3[1]; + u32x we_t = w3[2]; + u32x wf_t = w3[3]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) +{ + w0[0] = w0[0] ^ 0x36363636; + w0[1] = w0[1] ^ 0x36363636; + w0[2] = w0[2] ^ 0x36363636; + w0[3] = w0[3] ^ 0x36363636; + w1[0] = w1[0] ^ 0x36363636; + w1[1] = w1[1] ^ 0x36363636; + w1[2] = w1[2] ^ 0x36363636; + w1[3] = w1[3] ^ 0x36363636; + w2[0] = w2[0] ^ 0x36363636; + w2[1] = w2[1] ^ 0x36363636; + w2[2] = w2[2] ^ 0x36363636; + w2[3] = w2[3] ^ 0x36363636; + w3[0] = w3[0] ^ 0x36363636; + w3[1] = w3[1] ^ 0x36363636; + w3[2] = w3[2] ^ 0x36363636; + w3[3] = w3[3] ^ 0x36363636; + + ipad[0] = SHA256M_A; + ipad[1] = SHA256M_B; + ipad[2] = SHA256M_C; + ipad[3] = SHA256M_D; + ipad[4] = SHA256M_E; + ipad[5] = SHA256M_F; + ipad[6] = SHA256M_G; + ipad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, ipad); + + w0[0] = w0[0] ^ 0x6a6a6a6a; + w0[1] = w0[1] ^ 0x6a6a6a6a; + w0[2] = w0[2] ^ 0x6a6a6a6a; + w0[3] = w0[3] ^ 0x6a6a6a6a; + w1[0] = w1[0] ^ 0x6a6a6a6a; + w1[1] = w1[1] ^ 0x6a6a6a6a; + w1[2] = w1[2] ^ 0x6a6a6a6a; + w1[3] = w1[3] ^ 0x6a6a6a6a; + w2[0] = w2[0] ^ 0x6a6a6a6a; + w2[1] = w2[1] ^ 0x6a6a6a6a; + w2[2] = w2[2] ^ 0x6a6a6a6a; + w2[3] = w2[3] ^ 0x6a6a6a6a; + w3[0] = w3[0] ^ 0x6a6a6a6a; + w3[1] = w3[1] ^ 0x6a6a6a6a; + w3[2] = w3[2] ^ 0x6a6a6a6a; + w3[3] = w3[3] ^ 0x6a6a6a6a; + + opad[0] = SHA256M_A; + opad[1] = SHA256M_B; + opad[2] = SHA256M_C; + opad[3] = SHA256M_D; + opad[4] = SHA256M_E; + opad[5] = SHA256M_F; + opad[6] = SHA256M_G; + opad[7] = SHA256M_H; + + sha256_transform (w0, w1, w2, w3, opad); +} + +void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) +{ + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha256_transform (w0, w1, w2, w3, digest); + + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = digest[5]; + w1[2] = digest[6]; + w1[3] = digest[7]; + w2[0] = 0x80000000; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (64 + 32) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha256_transform (w0, w1, w2, w3, digest); +} + +void m01460m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + pw_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +void m01460s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u32x ipad[8]; + u32x opad[8]; + + hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (64 + pw_len) * 8; + + u32x digest[8]; + + hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + } +} + +__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01460_a3-pure.cl b/OpenCL/m01460_a3-pure.cl deleted file mode 100644 index 646ed4732..000000000 --- a/OpenCL/m01460_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha256.cl" - -__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_vector_t ctx0; - - sha256_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_hmac_ctx_vector_t ctx = ctx0; - - sha256_hmac_update_vector (&ctx, w, pw_len); - - sha256_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha256_hmac_ctx_vector_t ctx0; - - sha256_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha256_hmac_ctx_vector_t ctx = ctx0; - - sha256_hmac_update_vector (&ctx, w, pw_len); - - sha256_hmac_final_vector (&ctx); - - const u32x r0 = ctx.opad.h[DGST_R0]; - const u32x r1 = ctx.opad.h[DGST_R1]; - const u32x r2 = ctx.opad.h[DGST_R2]; - const u32x r3 = ctx.opad.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01460_a3.cl b/OpenCL/m01460_a3.cl index 156b75bcc..646ed4732 100644 --- a/OpenCL/m01460_a3.cl +++ b/OpenCL/m01460_a3.cl @@ -11,388 +11,92 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha256.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -void sha256_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u32x digest[8]) -{ - u32x a = digest[0]; - u32x b = digest[1]; - u32x c = digest[2]; - u32x d = digest[3]; - u32x e = digest[4]; - u32x f = digest[5]; - u32x g = digest[6]; - u32x h = digest[7]; - - u32x w0_t = w0[0]; - u32x w1_t = w0[1]; - u32x w2_t = w0[2]; - u32x w3_t = w0[3]; - u32x w4_t = w1[0]; - u32x w5_t = w1[1]; - u32x w6_t = w1[2]; - u32x w7_t = w1[3]; - u32x w8_t = w2[0]; - u32x w9_t = w2[1]; - u32x wa_t = w2[2]; - u32x wb_t = w2[3]; - u32x wc_t = w3[0]; - u32x wd_t = w3[1]; - u32x we_t = w3[2]; - u32x wf_t = w3[3]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha256_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8]) -{ - w0[0] = w0[0] ^ 0x36363636; - w0[1] = w0[1] ^ 0x36363636; - w0[2] = w0[2] ^ 0x36363636; - w0[3] = w0[3] ^ 0x36363636; - w1[0] = w1[0] ^ 0x36363636; - w1[1] = w1[1] ^ 0x36363636; - w1[2] = w1[2] ^ 0x36363636; - w1[3] = w1[3] ^ 0x36363636; - w2[0] = w2[0] ^ 0x36363636; - w2[1] = w2[1] ^ 0x36363636; - w2[2] = w2[2] ^ 0x36363636; - w2[3] = w2[3] ^ 0x36363636; - w3[0] = w3[0] ^ 0x36363636; - w3[1] = w3[1] ^ 0x36363636; - w3[2] = w3[2] ^ 0x36363636; - w3[3] = w3[3] ^ 0x36363636; - - ipad[0] = SHA256M_A; - ipad[1] = SHA256M_B; - ipad[2] = SHA256M_C; - ipad[3] = SHA256M_D; - ipad[4] = SHA256M_E; - ipad[5] = SHA256M_F; - ipad[6] = SHA256M_G; - ipad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, ipad); - - w0[0] = w0[0] ^ 0x6a6a6a6a; - w0[1] = w0[1] ^ 0x6a6a6a6a; - w0[2] = w0[2] ^ 0x6a6a6a6a; - w0[3] = w0[3] ^ 0x6a6a6a6a; - w1[0] = w1[0] ^ 0x6a6a6a6a; - w1[1] = w1[1] ^ 0x6a6a6a6a; - w1[2] = w1[2] ^ 0x6a6a6a6a; - w1[3] = w1[3] ^ 0x6a6a6a6a; - w2[0] = w2[0] ^ 0x6a6a6a6a; - w2[1] = w2[1] ^ 0x6a6a6a6a; - w2[2] = w2[2] ^ 0x6a6a6a6a; - w2[3] = w2[3] ^ 0x6a6a6a6a; - w3[0] = w3[0] ^ 0x6a6a6a6a; - w3[1] = w3[1] ^ 0x6a6a6a6a; - w3[2] = w3[2] ^ 0x6a6a6a6a; - w3[3] = w3[3] ^ 0x6a6a6a6a; - - opad[0] = SHA256M_A; - opad[1] = SHA256M_B; - opad[2] = SHA256M_C; - opad[3] = SHA256M_D; - opad[4] = SHA256M_E; - opad[5] = SHA256M_F; - opad[6] = SHA256M_G; - opad[7] = SHA256M_H; - - sha256_transform (w0, w1, w2, w3, opad); -} - -void hmac_sha256_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u32x ipad[8], u32x opad[8], u32x digest[8]) -{ - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha256_transform (w0, w1, w2, w3, digest); - - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = digest[5]; - w1[2] = digest[6]; - w1[3] = digest[7]; - w2[0] = 0x80000000; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (64 + 32) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha256_transform (w0, w1, w2, w3, digest); -} - -void m01460m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01460_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32x s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u32x ipad[8]; - u32x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha256_hmac_ctx_vector_t ctx0; + + sha256_hmac_init_vector (&ctx0, s, salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + pw_len) * 8; + w[0] = w0; - u32x digest[8]; + sha256_hmac_ctx_vector_t ctx = ctx0; - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha256_hmac_update_vector (&ctx, w, pw_len); - COMPARE_M_SIMD (digest[3], digest[7], digest[2], digest[6]); + sha256_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01460s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01460_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u32x ipad[8]; - u32x opad[8]; - - hmac_sha256_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + if (gid >= gid_max) return; /** * digest @@ -406,321 +110,65 @@ void m01460s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha256_hmac_ctx_vector_t ctx0; + + sha256_hmac_init_vector (&ctx0, s, salt_len); + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (64 + pw_len) * 8; + w[0] = w0; - u32x digest[8]; + sha256_hmac_ctx_vector_t ctx = ctx0; - hmac_sha256_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha256_hmac_update_vector (&ctx, w, pw_len); - COMPARE_S_SIMD (digest[3], digest[7], digest[2], digest[6]); + sha256_hmac_final_vector (&ctx); + + const u32x r0 = ctx.opad.h[DGST_R0]; + const u32x r1 = ctx.opad.h[DGST_R1]; + const u32x r2 = ctx.opad.h[DGST_R2]; + const u32x r3 = ctx.opad.h[DGST_R3]; + + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01460_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01460_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01460_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01460_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01460_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01460_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01460s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01600-optimized.cl b/OpenCL/m01600-optimized.cl new file mode 100644 index 000000000..fd5c4f293 --- /dev/null +++ b/OpenCL/m01600-optimized.cl @@ -0,0 +1,1139 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +#define md5apr1_magic0 0x72706124u +#define md5apr1_magic1 0x00002431u + +void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + + u32 w0_t = w0[0]; + u32 w1_t = w0[1]; + u32 w2_t = w0[2]; + u32 w3_t = w0[3]; + u32 w4_t = w1[0]; + u32 w5_t = w1[1]; + u32 w6_t = w1[2]; + u32 w7_t = w1[3]; + u32 w8_t = w2[0]; + u32 w9_t = w2[1]; + u32 wa_t = w2[2]; + u32 wb_t = w2[3]; + u32 wc_t = w3[0]; + u32 wd_t = w3[1]; + u32 we_t = w3[2]; + u32 wf_t = 0; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0x80; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0x80, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + break; + case 10: block2[2] |= tmp0; + block2[3] = tmp1; + block3[0] = tmp2; + break; + case 11: block2[3] |= tmp0; + block3[0] = tmp1; + block3[1] = tmp2; + break; + } +} + +void append_sign (u32 block0[4], u32 block1[4], const u32 block_len) +{ + switch (block_len) + { + case 0: + block0[0] = md5apr1_magic0; + block0[1] = md5apr1_magic1; + break; + + case 1: + block0[0] = block0[0] | md5apr1_magic0 << 8u; + block0[1] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; + block0[2] = md5apr1_magic1 >> 24u; + break; + + case 2: + block0[0] = block0[0] | md5apr1_magic0 << 16u; + block0[1] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; + block0[2] = md5apr1_magic1 >> 16u; + break; + + case 3: + block0[0] = block0[0] | md5apr1_magic0 << 24u; + block0[1] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; + block0[2] = md5apr1_magic1 >> 8u; + break; + + case 4: + block0[1] = md5apr1_magic0; + block0[2] = md5apr1_magic1; + break; + + case 5: + block0[1] = block0[1] | md5apr1_magic0 << 8u; + block0[2] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; + block0[3] = md5apr1_magic1 >> 24u; + break; + + case 6: + block0[1] = block0[1] | md5apr1_magic0 << 16u; + block0[2] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; + block0[3] = md5apr1_magic1 >> 16u; + break; + + case 7: + block0[1] = block0[1] | md5apr1_magic0 << 24u; + block0[2] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; + block0[3] = md5apr1_magic1 >> 8u; + break; + + case 8: + block0[2] = md5apr1_magic0; + block0[3] = md5apr1_magic1; + break; + + case 9: + block0[2] = block0[2] | md5apr1_magic0 << 8u; + block0[3] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; + block1[0] = md5apr1_magic1 >> 24u; + break; + + case 10: + block0[2] = block0[2] | md5apr1_magic0 << 16u; + block0[3] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; + block1[0] = md5apr1_magic1 >> 16u; + break; + + case 11: + block0[2] = block0[2] | md5apr1_magic0 << 24u; + block0[3] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; + block1[0] = md5apr1_magic1 >> 8u; + break; + + case 12: + block0[3] = md5apr1_magic0; + block1[0] = md5apr1_magic1; + break; + + case 13: + block0[3] = block0[3] | md5apr1_magic0 << 8u; + block1[0] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; + block1[1] = md5apr1_magic1 >> 24u; + break; + + case 14: + block0[3] = block0[3] | md5apr1_magic0 << 16u; + block1[0] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; + block1[1] = md5apr1_magic1 >> 16u; + break; + + case 15: + block0[3] = block0[3] | md5apr1_magic0 << 24u; + block1[0] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; + block1[1] = md5apr1_magic1 >> 8u; + break; + } +} + +void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) +{ + switch (block_len) + { + case 0: + block0[0] = append; + break; + + case 1: + block0[0] = block0[0] | append << 8; + break; + + case 2: + block0[0] = block0[0] | append << 16; + break; + + case 3: + block0[0] = block0[0] | append << 24; + break; + + case 4: + block0[1] = append; + break; + + case 5: + block0[1] = block0[1] | append << 8; + break; + + case 6: + block0[1] = block0[1] | append << 16; + break; + + case 7: + block0[1] = block0[1] | append << 24; + break; + + case 8: + block0[2] = append; + break; + + case 9: + block0[2] = block0[2] | append << 8; + break; + + case 10: + block0[2] = block0[2] | append << 16; + break; + + case 11: + block0[2] = block0[2] | append << 24; + break; + + case 12: + block0[3] = append; + break; + + case 13: + block0[3] = block0[3] | append << 8; + break; + + case 14: + block0[3] = block0[3] | append << 16; + break; + + case 15: + block0[3] = block0[3] | append << 24; + break; + + case 16: + block1[0] = append; + break; + + case 17: + block1[0] = block1[0] | append << 8; + break; + + case 18: + block1[0] = block1[0] | append << 16; + break; + + case 19: + block1[0] = block1[0] | append << 24; + break; + + case 20: + block1[1] = append; + break; + + case 21: + block1[1] = block1[1] | append << 8; + break; + + case 22: + block1[1] = block1[1] | append << 16; + break; + + case 23: + block1[1] = block1[1] | append << 24; + break; + + case 24: + block1[2] = append; + break; + + case 25: + block1[2] = block1[2] | append << 8; + break; + + case 26: + block1[2] = block1[2] | append << 16; + break; + + case 27: + block1[2] = block1[2] | append << 24; + break; + + case 28: + block1[3] = append; + break; + + case 29: + block1[3] = block1[3] | append << 8; + break; + + case 30: + block1[3] = block1[3] | append << 16; + break; + + case 31: + block1[3] = block1[3] | append << 24; + break; + + case 32: + block2[0] = append; + break; + + case 33: + block2[0] = block2[0] | append << 8; + break; + + case 34: + block2[0] = block2[0] | append << 16; + break; + + case 35: + block2[0] = block2[0] | append << 24; + break; + + case 36: + block2[1] = append; + break; + + case 37: + block2[1] = block2[1] | append << 8; + break; + + case 38: + block2[1] = block2[1] | append << 16; + break; + + case 39: + block2[1] = block2[1] | append << 24; + break; + + case 40: + block2[2] = append; + break; + + case 41: + block2[2] = block2[2] | append << 8; + break; + + case 42: + block2[2] = block2[2] | append << 16; + break; + + case 43: + block2[2] = block2[2] | append << 24; + break; + + case 44: + block2[3] = append; + break; + + case 45: + block2[3] = block2[3] | append << 8; + break; + + case 46: + block2[3] = block2[3] | append << 16; + break; + + case 47: + block2[3] = block2[3] | append << 24; + break; + + case 48: + block3[0] = append; + break; + + case 49: + block3[0] = block3[0] | append << 8; + break; + + case 50: + block3[0] = block3[0] | append << 16; + break; + + case 51: + block3[0] = block3[0] | append << 24; + break; + + case 52: + block3[1] = append; + break; + + case 53: + block3[1] = block3[1] | append << 8; + break; + + case 54: + block3[1] = block3[1] | append << 16; + break; + + case 55: + block3[1] = block3[1] | append << 24; + break; + + case 56: + block3[2] = append; + break; + } +} + +__kernel void m01600_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * init + */ + + //memcat16 (block0, block1, block2, block3, block_len, w0); + //block_len += pw_len; + + u32 block_len = pw_len; + + u32 block0[4]; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + u32 digest[4]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + /* The password first, since that is what is most unknown */ + /* Then our magic string */ + /* Then the raw salt */ + /* Then just as many characters of the MD5(pw,salt,pw) */ + + //memcat16 (block0, block1, block2, block3, block_len, w); + //block_len += pw_len; + + block_len = pw_len; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + append_sign (block0, block1, block_len); + + block_len += 6; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + truncate_block_4x4_le (digest, pw_len); + + memcat16 (block0, block1, block2, block3, block_len, digest); + + block_len += pw_len; + + /* Then something really weird... */ + + u32 append = block0[0] & 0xFF; + + for (u32 j = pw_len; j; j >>= 1) + { + if ((j & 1) == 0) + { + append_1st (block0, block1, block2, block3, block_len, append); + } + + block_len++; + } + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m01600_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + u32 w0_x80[4]; + + w0_x80[0] = w0[0]; + w0_x80[1] = w0[1]; + w0_x80[2] = w0[2]; + w0_x80[3] = w0[3]; + + append_0x80_1x4 (w0_x80, pw_len); + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + u32 digest[4]; + + digest[0] = tmps[gid].digest_buf[0]; + digest[1] = tmps[gid].digest_buf[1]; + digest[2] = tmps[gid].digest_buf[2]; + digest[3] = tmps[gid].digest_buf[3]; + + /** + * loop + */ + + /* and now, just to make sure things don't run too fast */ + + u32 block_len; + + u32 block0[4]; + + block0[0] = 0; + block0[1] = 0; + block0[2] = 0; + block0[3] = 0; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + block3[0] = 0; + block3[1] = 0; + + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 1 : 0; + const u32 j7 = (j % 7) ? 1 : 0; + + if (j1) + { + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block_len = pw_len; + + if (j3) + { + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + } + + if (j7) + { + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + + memcat16_x80 (block0, block1, block2, block3, block_len, digest); + + block_len += 16; + } + else + { + block0[0] = digest[0]; + block0[1] = digest[1]; + block0[2] = digest[2]; + block0[3] = digest[3]; + + block_len = 16; + + if (j3 && j7) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + else if (j3) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + } + else if (j7) + { + block1[0] = w0[0]; + block1[1] = w0[1]; + block1[2] = w0[2]; + block1[3] = w0[3]; + + block_len += pw_len; + } + + memcat16 (block0, block1, block2, block3, block_len, w0_x80); + + block_len += pw_len; + } + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + } + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m01600_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 r0 = tmps[gid].digest_buf[DGST_R0]; + const u32 r1 = tmps[gid].digest_buf[DGST_R1]; + const u32 r2 = tmps[gid].digest_buf[DGST_R2]; + const u32 r3 = tmps[gid].digest_buf[DGST_R3]; + + #define il_pos 0 + + #include COMPARE_M +} diff --git a/OpenCL/m01600-pure.cl b/OpenCL/m01600-pure.cl deleted file mode 100644 index dc110a813..000000000 --- a/OpenCL/m01600-pure.cl +++ /dev/null @@ -1,263 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_md5.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -#define md5apr1_magic0 0x72706124u -#define md5apr1_magic1 0x00002431u - -__kernel void m01600_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * prepare - */ - - md5_ctx_t md5_ctx1; - - md5_init (&md5_ctx1); - - md5_update (&md5_ctx1, w, pw_len); - - md5_update (&md5_ctx1, s, salt_len); - - md5_update (&md5_ctx1, w, pw_len); - - md5_final (&md5_ctx1); - - u32 final[16] = { 0 }; - - final[0] = md5_ctx1.h[0]; - final[1] = md5_ctx1.h[1]; - final[2] = md5_ctx1.h[2]; - final[3] = md5_ctx1.h[3]; - - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - md5_update (&md5_ctx, w, pw_len); - - u32 m[16] = { 0 }; - - m[0] = md5apr1_magic0; - m[1] = md5apr1_magic1; - - md5_update (&md5_ctx, m, 6); - - md5_update (&md5_ctx, s, salt_len); - - int pl; - - for (pl = pw_len; pl > 16; pl -= 16) - { - md5_update (&md5_ctx, final, 16); - } - - truncate_block_4x4_le (final, pl); - - md5_update (&md5_ctx, final, pl); - - /* Then something really weird... */ - - for (int i = pw_len; i != 0; i >>= 1) - { - u32 t[16] = { 0 }; - - if (i & 1) - { - t[0] = 0; - } - else - { - t[0] = w[0] & 0xff; - } - - md5_update (&md5_ctx, t, 1); - } - - md5_final (&md5_ctx); - - tmps[gid].digest_buf[0] = md5_ctx.h[0]; - tmps[gid].digest_buf[1] = md5_ctx.h[1]; - tmps[gid].digest_buf[2] = md5_ctx.h[2]; - tmps[gid].digest_buf[3] = md5_ctx.h[3]; -} - -__kernel void m01600_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * digest - */ - - u32 digest[16] = { 0 }; - - digest[0] = tmps[gid].digest_buf[0]; - digest[1] = tmps[gid].digest_buf[1]; - digest[2] = tmps[gid].digest_buf[2]; - digest[3] = tmps[gid].digest_buf[3]; - - /** - * loop - */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - if (j & 1) - { - md5_update (&md5_ctx, w, pw_len); - } - else - { - md5_update (&md5_ctx, digest, 16); - } - - if (j % 3) - { - md5_update (&md5_ctx, s, salt_len); - } - - if (j % 7) - { - md5_update (&md5_ctx, w, pw_len); - } - - if (j & 1) - { - md5_update (&md5_ctx, digest, 16); - } - else - { - md5_update (&md5_ctx, w, pw_len); - } - - md5_final (&md5_ctx); - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - } - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; -} - -__kernel void m01600_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - /** - * digest - */ - - const u32 r0 = tmps[gid].digest_buf[DGST_R0]; - const u32 r1 = tmps[gid].digest_buf[DGST_R1]; - const u32 r2 = tmps[gid].digest_buf[DGST_R2]; - const u32 r3 = tmps[gid].digest_buf[DGST_R3]; - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m01600.cl b/OpenCL/m01600.cl index fd5c4f293..dc110a813 100644 --- a/OpenCL/m01600.cl +++ b/OpenCL/m01600.cl @@ -3,11 +3,14 @@ * License.....: MIT */ +//#define NEW_SIMD_CODE + #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_md5.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" @@ -15,752 +18,6 @@ #define md5apr1_magic0 0x72706124u #define md5apr1_magic1 0x00002431u -void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - - u32 w0_t = w0[0]; - u32 w1_t = w0[1]; - u32 w2_t = w0[2]; - u32 w3_t = w0[3]; - u32 w4_t = w1[0]; - u32 w5_t = w1[1]; - u32 w6_t = w1[2]; - u32 w7_t = w1[3]; - u32 w8_t = w2[0]; - u32 w9_t = w2[1]; - u32 wa_t = w2[2]; - u32 wb_t = w2[3]; - u32 wc_t = w3[0]; - u32 wd_t = w3[1]; - u32 we_t = w3[2]; - u32 wf_t = 0; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0x80; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0x80, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - break; - case 10: block2[2] |= tmp0; - block2[3] = tmp1; - block3[0] = tmp2; - break; - case 11: block2[3] |= tmp0; - block3[0] = tmp1; - block3[1] = tmp2; - break; - } -} - -void append_sign (u32 block0[4], u32 block1[4], const u32 block_len) -{ - switch (block_len) - { - case 0: - block0[0] = md5apr1_magic0; - block0[1] = md5apr1_magic1; - break; - - case 1: - block0[0] = block0[0] | md5apr1_magic0 << 8u; - block0[1] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; - block0[2] = md5apr1_magic1 >> 24u; - break; - - case 2: - block0[0] = block0[0] | md5apr1_magic0 << 16u; - block0[1] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; - block0[2] = md5apr1_magic1 >> 16u; - break; - - case 3: - block0[0] = block0[0] | md5apr1_magic0 << 24u; - block0[1] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; - block0[2] = md5apr1_magic1 >> 8u; - break; - - case 4: - block0[1] = md5apr1_magic0; - block0[2] = md5apr1_magic1; - break; - - case 5: - block0[1] = block0[1] | md5apr1_magic0 << 8u; - block0[2] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; - block0[3] = md5apr1_magic1 >> 24u; - break; - - case 6: - block0[1] = block0[1] | md5apr1_magic0 << 16u; - block0[2] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; - block0[3] = md5apr1_magic1 >> 16u; - break; - - case 7: - block0[1] = block0[1] | md5apr1_magic0 << 24u; - block0[2] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; - block0[3] = md5apr1_magic1 >> 8u; - break; - - case 8: - block0[2] = md5apr1_magic0; - block0[3] = md5apr1_magic1; - break; - - case 9: - block0[2] = block0[2] | md5apr1_magic0 << 8u; - block0[3] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; - block1[0] = md5apr1_magic1 >> 24u; - break; - - case 10: - block0[2] = block0[2] | md5apr1_magic0 << 16u; - block0[3] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; - block1[0] = md5apr1_magic1 >> 16u; - break; - - case 11: - block0[2] = block0[2] | md5apr1_magic0 << 24u; - block0[3] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; - block1[0] = md5apr1_magic1 >> 8u; - break; - - case 12: - block0[3] = md5apr1_magic0; - block1[0] = md5apr1_magic1; - break; - - case 13: - block0[3] = block0[3] | md5apr1_magic0 << 8u; - block1[0] = md5apr1_magic0 >> 24u | md5apr1_magic1 << 8u; - block1[1] = md5apr1_magic1 >> 24u; - break; - - case 14: - block0[3] = block0[3] | md5apr1_magic0 << 16u; - block1[0] = md5apr1_magic0 >> 16u | md5apr1_magic1 << 16u; - block1[1] = md5apr1_magic1 >> 16u; - break; - - case 15: - block0[3] = block0[3] | md5apr1_magic0 << 24u; - block1[0] = md5apr1_magic0 >> 8u | md5apr1_magic1 << 24u; - block1[1] = md5apr1_magic1 >> 8u; - break; - } -} - -void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) -{ - switch (block_len) - { - case 0: - block0[0] = append; - break; - - case 1: - block0[0] = block0[0] | append << 8; - break; - - case 2: - block0[0] = block0[0] | append << 16; - break; - - case 3: - block0[0] = block0[0] | append << 24; - break; - - case 4: - block0[1] = append; - break; - - case 5: - block0[1] = block0[1] | append << 8; - break; - - case 6: - block0[1] = block0[1] | append << 16; - break; - - case 7: - block0[1] = block0[1] | append << 24; - break; - - case 8: - block0[2] = append; - break; - - case 9: - block0[2] = block0[2] | append << 8; - break; - - case 10: - block0[2] = block0[2] | append << 16; - break; - - case 11: - block0[2] = block0[2] | append << 24; - break; - - case 12: - block0[3] = append; - break; - - case 13: - block0[3] = block0[3] | append << 8; - break; - - case 14: - block0[3] = block0[3] | append << 16; - break; - - case 15: - block0[3] = block0[3] | append << 24; - break; - - case 16: - block1[0] = append; - break; - - case 17: - block1[0] = block1[0] | append << 8; - break; - - case 18: - block1[0] = block1[0] | append << 16; - break; - - case 19: - block1[0] = block1[0] | append << 24; - break; - - case 20: - block1[1] = append; - break; - - case 21: - block1[1] = block1[1] | append << 8; - break; - - case 22: - block1[1] = block1[1] | append << 16; - break; - - case 23: - block1[1] = block1[1] | append << 24; - break; - - case 24: - block1[2] = append; - break; - - case 25: - block1[2] = block1[2] | append << 8; - break; - - case 26: - block1[2] = block1[2] | append << 16; - break; - - case 27: - block1[2] = block1[2] | append << 24; - break; - - case 28: - block1[3] = append; - break; - - case 29: - block1[3] = block1[3] | append << 8; - break; - - case 30: - block1[3] = block1[3] | append << 16; - break; - - case 31: - block1[3] = block1[3] | append << 24; - break; - - case 32: - block2[0] = append; - break; - - case 33: - block2[0] = block2[0] | append << 8; - break; - - case 34: - block2[0] = block2[0] | append << 16; - break; - - case 35: - block2[0] = block2[0] | append << 24; - break; - - case 36: - block2[1] = append; - break; - - case 37: - block2[1] = block2[1] | append << 8; - break; - - case 38: - block2[1] = block2[1] | append << 16; - break; - - case 39: - block2[1] = block2[1] | append << 24; - break; - - case 40: - block2[2] = append; - break; - - case 41: - block2[2] = block2[2] | append << 8; - break; - - case 42: - block2[2] = block2[2] | append << 16; - break; - - case 43: - block2[2] = block2[2] | append << 24; - break; - - case 44: - block2[3] = append; - break; - - case 45: - block2[3] = block2[3] | append << 8; - break; - - case 46: - block2[3] = block2[3] | append << 16; - break; - - case 47: - block2[3] = block2[3] | append << 24; - break; - - case 48: - block3[0] = append; - break; - - case 49: - block3[0] = block3[0] | append << 8; - break; - - case 50: - block3[0] = block3[0] | append << 16; - break; - - case 51: - block3[0] = block3[0] | append << 24; - break; - - case 52: - block3[1] = append; - break; - - case 53: - block3[1] = block3[1] | append << 8; - break; - - case 54: - block3[1] = block3[1] | append << 16; - break; - - case 55: - block3[1] = block3[1] | append << 24; - break; - - case 56: - block3[2] = append; - break; - } -} - __kernel void m01600_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -771,157 +28,109 @@ __kernel void m01600_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * init */ - //memcat16 (block0, block1, block2, block3, block_len, w0); - //block_len += pw_len; + const u32 pw_len = pws[gid].pw_len; - u32 block_len = pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 block0[4]; + u32 w[64] = { 0 }; - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - u32 block1[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 block2[4]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + u32 s[64] = { 0 }; - u32 block3[4]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + barrier (CLK_GLOBAL_MEM_FENCE); + } - memcat8 (block0, block1, block2, block3, block_len, salt_buf); + /** + * prepare + */ - block_len += salt_len; + md5_ctx_t md5_ctx1; - memcat16 (block0, block1, block2, block3, block_len, w0); + md5_init (&md5_ctx1); - block_len += pw_len; + md5_update (&md5_ctx1, w, pw_len); - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_update (&md5_ctx1, s, salt_len); - block3[2] = block_len * 8; + md5_update (&md5_ctx1, w, pw_len); - u32 digest[4]; + md5_final (&md5_ctx1); - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + u32 final[16] = { 0 }; - md5_transform (block0, block1, block2, block3, digest); + final[0] = md5_ctx1.h[0]; + final[1] = md5_ctx1.h[1]; + final[2] = md5_ctx1.h[2]; + final[3] = md5_ctx1.h[3]; - /* The password first, since that is what is most unknown */ - /* Then our magic string */ - /* Then the raw salt */ - /* Then just as many characters of the MD5(pw,salt,pw) */ + md5_ctx_t md5_ctx; - //memcat16 (block0, block1, block2, block3, block_len, w); - //block_len += pw_len; + md5_init (&md5_ctx); - block_len = pw_len; + md5_update (&md5_ctx, w, pw_len); - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + u32 m[16] = { 0 }; - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + m[0] = md5apr1_magic0; + m[1] = md5apr1_magic1; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + md5_update (&md5_ctx, m, 6); - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + md5_update (&md5_ctx, s, salt_len); - append_sign (block0, block1, block_len); + int pl; - block_len += 6; + for (pl = pw_len; pl > 16; pl -= 16) + { + md5_update (&md5_ctx, final, 16); + } - memcat8 (block0, block1, block2, block3, block_len, salt_buf); + truncate_block_4x4_le (final, pl); - block_len += salt_len; - - truncate_block_4x4_le (digest, pw_len); - - memcat16 (block0, block1, block2, block3, block_len, digest); - - block_len += pw_len; + md5_update (&md5_ctx, final, pl); /* Then something really weird... */ - u32 append = block0[0] & 0xFF; - - for (u32 j = pw_len; j; j >>= 1) + for (int i = pw_len; i != 0; i >>= 1) { - if ((j & 1) == 0) + u32 t[16] = { 0 }; + + if (i & 1) { - append_1st (block0, block1, block2, block3, block_len, append); + t[0] = 0; + } + else + { + t[0] = w[0] & 0xff; } - block_len++; + md5_update (&md5_ctx, t, 1); } - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_final (&md5_ctx); - block3[2] = block_len * 8; - - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform (block0, block1, block2, block3, digest); - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; + tmps[gid].digest_buf[0] = md5_ctx.h[0]; + tmps[gid].digest_buf[1] = md5_ctx.h[1]; + tmps[gid].digest_buf[2] = md5_ctx.h[2]; + tmps[gid].digest_buf[3] = md5_ctx.h[3]; } __kernel void m01600_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -934,40 +143,41 @@ __kernel void m01600_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; + /** + * init + */ const u32 pw_len = pws[gid].pw_len; - u32 w0_x80[4]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - w0_x80[0] = w0[0]; - w0_x80[1] = w0[1]; - w0_x80[2] = w0[2]; - w0_x80[3] = w0[3]; + u32 w[64] = { 0 }; - append_0x80_1x4 (w0_x80, pw_len); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * digest */ - u32 digest[4]; + u32 digest[16] = { 0 }; digest[0] = tmps[gid].digest_buf[0]; digest[1] = tmps[gid].digest_buf[1]; @@ -978,132 +188,46 @@ __kernel void m01600_loop (__global pw_t *pws, __global const kernel_rule_t *rul * loop */ - /* and now, just to make sure things don't run too fast */ - - u32 block_len; - - u32 block0[4]; - - block0[0] = 0; - block0[1] = 0; - block0[2] = 0; - block0[3] = 0; - - u32 block1[4]; - - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - - u32 block2[4]; - - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - - u32 block3[4]; - - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - block3[0] = 0; - block3[1] = 0; + md5_ctx_t md5_ctx; - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 1 : 0; - const u32 j7 = (j % 7) ? 1 : 0; + md5_init (&md5_ctx); - if (j1) + if (j & 1) { - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; - - block_len = pw_len; - - if (j3) - { - memcat8 (block0, block1, block2, block3, block_len, salt_buf); - - block_len += salt_len; - } - - if (j7) - { - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - - memcat16_x80 (block0, block1, block2, block3, block_len, digest); - - block_len += 16; + md5_update (&md5_ctx, w, pw_len); } - else + else { - block0[0] = digest[0]; - block0[1] = digest[1]; - block0[2] = digest[2]; - block0[3] = digest[3]; - - block_len = 16; - - if (j3 && j7) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - else if (j3) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - } - else if (j7) - { - block1[0] = w0[0]; - block1[1] = w0[1]; - block1[2] = w0[2]; - block1[3] = w0[3]; - - block_len += pw_len; - } - - memcat16 (block0, block1, block2, block3, block_len, w0_x80); - - block_len += pw_len; + md5_update (&md5_ctx, digest, 16); } - block3[2] = block_len * 8; + if (j % 3) + { + md5_update (&md5_ctx, s, salt_len); + } - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + if (j % 7) + { + md5_update (&md5_ctx, w, pw_len); + } - md5_transform (block0, block1, block2, block3, digest); + if (j & 1) + { + md5_update (&md5_ctx, digest, 16); + } + else + { + md5_update (&md5_ctx, w, pw_len); + } + + md5_final (&md5_ctx); + + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; } tmps[gid].digest_buf[0] = digest[0]; diff --git a/OpenCL/m01700_a0-optimized.cl b/OpenCL/m01700_a0-optimized.cl new file mode 100644 index 000000000..80f47b085 --- /dev/null +++ b/OpenCL/m01700_a0-optimized.cl @@ -0,0 +1,352 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01700_a0-pure.cl b/OpenCL/m01700_a0-pure.cl deleted file mode 100644 index 01649a999..000000000 --- a/OpenCL/m01700_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01700_a0.cl b/OpenCL/m01700_a0.cl index 80f47b085..01649a999 100644 --- a/OpenCL/m01700_a0.cl +++ b/OpenCL/m01700_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,262 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha512_init (&ctx); - /** - * sha512 - */ + sha512_update_swap (&ctx, w, pw_len); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_final (&ctx); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_len * 8; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -280,73 +91,44 @@ __kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha512_init (&ctx); - /** - * sha512 - */ + sha512_update_swap (&ctx, w, pw_len); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_final (&ctx); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_len * 8; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01700_a1-optimized.cl b/OpenCL/m01700_a1-optimized.cl new file mode 100644 index 000000000..c00c1f5ed --- /dev/null +++ b/OpenCL/m01700_a1-optimized.cl @@ -0,0 +1,466 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01700_a1-pure.cl b/OpenCL/m01700_a1-pure.cl deleted file mode 100644 index 5db180695..000000000 --- a/OpenCL/m01700_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01700_a1.cl b/OpenCL/m01700_a1.cl index c00c1f5ed..5db180695 100644 --- a/OpenCL/m01700_a1.cl +++ b/OpenCL/m01700_a1.cl @@ -3,327 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha512_ctx_t ctx0; - if (gid >= gid_max) return; + sha512_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -336,131 +78,33 @@ __kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01700_a3-optimized.cl b/OpenCL/m01700_a3-optimized.cl new file mode 100644 index 000000000..a2da35517 --- /dev/null +++ b/OpenCL/m01700_a3-optimized.cl @@ -0,0 +1,506 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +void m01700m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01700s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01700_a3-pure.cl b/OpenCL/m01700_a3-pure.cl deleted file mode 100644 index e35083c8b..000000000 --- a/OpenCL/m01700_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01700_a3.cl b/OpenCL/m01700_a3.cl index a2da35517..e35083c8b 100644 --- a/OpenCL/m01700_a3.cl +++ b/OpenCL/m01700_a3.cl @@ -11,144 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -void m01700m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01700_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -156,58 +53,35 @@ void m01700m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_final_vector (&ctx); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01700s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01700_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -221,11 +95,28 @@ void m01700s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -233,274 +124,21 @@ void m01700s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_final_vector (&ctx); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01700_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01700_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01700_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01700_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01700_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01700_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01700s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01710_a0-optimized.cl b/OpenCL/m01710_a0-optimized.cl new file mode 100644 index 000000000..21c9b0391 --- /dev/null +++ b/OpenCL/m01710_a0-optimized.cl @@ -0,0 +1,498 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01710_a0-pure.cl b/OpenCL/m01710_a0-pure.cl deleted file mode 100644 index 593be6b7d..000000000 --- a/OpenCL/m01710_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_swap (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_swap (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01710_a0.cl b/OpenCL/m01710_a0.cl index 21c9b0391..593be6b7d 100644 --- a/OpenCL/m01710_a0.cl +++ b/OpenCL/m01710_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,363 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - /** - * append salt - */ + sha512_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha512_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha512_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha512_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -381,118 +106,59 @@ __kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - /** - * append salt - */ + sha512_init (&ctx); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha512_update_swap (&ctx, w, pw_len); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + sha512_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + sha512_final (&ctx); - const u32x pw_salt_len = out_len + salt_len; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01710_a1-optimized.cl b/OpenCL/m01710_a1-optimized.cl new file mode 100644 index 000000000..432f43a4a --- /dev/null +++ b/OpenCL/m01710_a1-optimized.cl @@ -0,0 +1,616 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01710_a1-pure.cl b/OpenCL/m01710_a1-pure.cl deleted file mode 100644 index 485adbb32..000000000 --- a/OpenCL/m01710_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01710_a1.cl b/OpenCL/m01710_a1.cl index 432f43a4a..485adbb32 100644 --- a/OpenCL/m01710_a1.cl +++ b/OpenCL/m01710_a1.cl @@ -3,430 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha512_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -439,178 +93,48 @@ __kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha512_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01710_a3-optimized.cl b/OpenCL/m01710_a3-optimized.cl new file mode 100644 index 000000000..55b888fee --- /dev/null +++ b/OpenCL/m01710_a3-optimized.cl @@ -0,0 +1,557 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +void m01710m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01710s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01710_a3-pure.cl b/OpenCL/m01710_a3-pure.cl deleted file mode 100644 index 6da78255b..000000000 --- a/OpenCL/m01710_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_update_vector (&ctx, s, salt_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_update_vector (&ctx, s, salt_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01710_a3.cl b/OpenCL/m01710_a3.cl index 55b888fee..6da78255b 100644 --- a/OpenCL/m01710_a3.cl +++ b/OpenCL/m01710_a3.cl @@ -11,195 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -void m01710m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + u32x w[64] = { 0 }; - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); - w[15] = pw_salt_len * 8; + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -207,58 +66,37 @@ void m01710m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_update_vector (&ctx, s, salt_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01710s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -272,11 +110,41 @@ void m01710s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -284,274 +152,23 @@ void m01710s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_update_vector (&ctx, s, salt_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01710s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01720_a0-optimized.cl b/OpenCL/m01720_a0-optimized.cl new file mode 100644 index 000000000..54a1ea172 --- /dev/null +++ b/OpenCL/m01720_a0-optimized.cl @@ -0,0 +1,458 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01720_a0-pure.cl b/OpenCL/m01720_a0-pure.cl deleted file mode 100644 index adcf3501b..000000000 --- a/OpenCL/m01720_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx = ctx0; - - sha512_update_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx = ctx0; - - sha512_update_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01720_a0.cl b/OpenCL/m01720_a0.cl index 54a1ea172..adcf3501b 100644 --- a/OpenCL/m01720_a0.cl +++ b/OpenCL/m01720_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,343 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx = ctx0; - /** - * prepend salt - */ + sha512_update_swap (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + sha512_final (&ctx); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -361,98 +95,48 @@ __kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx = ctx0; - /** - * prepend salt - */ + sha512_update_swap (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + sha512_final (&ctx); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01720_a1-optimized.cl b/OpenCL/m01720_a1-optimized.cl new file mode 100644 index 000000000..c590cb13c --- /dev/null +++ b/OpenCL/m01720_a1-optimized.cl @@ -0,0 +1,572 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01720_a1-pure.cl b/OpenCL/m01720_a1-pure.cl deleted file mode 100644 index e1782f2f0..000000000 --- a/OpenCL/m01720_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01720_a1.cl b/OpenCL/m01720_a1.cl index c590cb13c..e1782f2f0 100644 --- a/OpenCL/m01720_a1.cl +++ b/OpenCL/m01720_a1.cl @@ -3,408 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha512_ctx_t ctx0; - if (gid >= gid_max) return; + sha512_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -417,156 +80,35 @@ __kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha512_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01720_a3-optimized.cl b/OpenCL/m01720_a3-optimized.cl new file mode 100644 index 000000000..4e57bfcf2 --- /dev/null +++ b/OpenCL/m01720_a3-optimized.cl @@ -0,0 +1,722 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +void m01720m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01720s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01720_a3-pure.cl b/OpenCL/m01720_a3-pure.cl deleted file mode 100644 index e6bb100d3..000000000 --- a/OpenCL/m01720_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector_from_scalar (&ctx, &ctx0); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector_from_scalar (&ctx, &ctx0); - - sha512_update_vector (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01720_a3.cl b/OpenCL/m01720_a3.cl index 4e57bfcf2..e6bb100d3 100644 --- a/OpenCL/m01720_a3.cl +++ b/OpenCL/m01720_a3.cl @@ -11,284 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -void m01720m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01720_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha512_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha512_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha512 - */ + sha512_ctx_vector_t ctx; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; + sha512_update_vector (&ctx, w, pw_len); - u64x digest[8]; + sha512_final_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01720s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01720_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -303,420 +102,55 @@ void m01720s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha512_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha512_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha512 - */ + sha512_ctx_vector_t ctx; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; + sha512_update_vector (&ctx, w, pw_len); - u64x digest[8]; + sha512_final_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01720_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01720_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01720_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01720_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01720_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01720_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01720s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01730_a0-optimized.cl b/OpenCL/m01730_a0-optimized.cl new file mode 100644 index 000000000..ed221ae2c --- /dev/null +++ b/OpenCL/m01730_a0-optimized.cl @@ -0,0 +1,508 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); + + const u32x pw_salt_len = out_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01730_a0-pure.cl b/OpenCL/m01730_a0-pure.cl deleted file mode 100644 index f4a8951a8..000000000 --- a/OpenCL/m01730_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_utf16le_swap (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update_utf16le_swap (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01730_a0.cl b/OpenCL/m01730_a0.cl index ed221ae2c..f4a8951a8 100644 --- a/OpenCL/m01730_a0.cl +++ b/OpenCL/m01730_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,368 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha512_init (&ctx); - const u32x out_len2 = out_len * 2; + sha512_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha512_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha512_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -386,123 +106,59 @@ __kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha512_init (&ctx); - const u32x out_len2 = out_len * 2; + sha512_update_utf16le_swap (&ctx, w, pw_len); - /** - * append salt - */ + sha512_update (&ctx, s, salt_len); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + sha512_final (&ctx); - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len2); - - const u32x pw_salt_len = out_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01730_a1-optimized.cl b/OpenCL/m01730_a1-optimized.cl new file mode 100644 index 000000000..f7d2fdfb8 --- /dev/null +++ b/OpenCL/m01730_a1-optimized.cl @@ -0,0 +1,626 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01730_a1-pure.cl b/OpenCL/m01730_a1-pure.cl deleted file mode 100644 index 7529dd71d..000000000 --- a/OpenCL/m01730_a1-pure.cl +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01730_a1.cl b/OpenCL/m01730_a1.cl index f7d2fdfb8..7529dd71d 100644 --- a/OpenCL/m01730_a1.cl +++ b/OpenCL/m01730_a1.cl @@ -3,435 +3,84 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha512_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -444,183 +93,48 @@ __kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + sha512_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len2); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01730_a3-optimized.cl b/OpenCL/m01730_a3-optimized.cl new file mode 100644 index 000000000..67a3d98d3 --- /dev/null +++ b/OpenCL/m01730_a3-optimized.cl @@ -0,0 +1,557 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +void m01730m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + + w[ 0] |= swap32_S (salt_buf0[0]); + w[ 1] |= swap32_S (salt_buf0[1]); + w[ 2] |= swap32_S (salt_buf0[2]); + w[ 3] |= swap32_S (salt_buf0[3]); + w[ 4] |= swap32_S (salt_buf1[0]); + w[ 5] |= swap32_S (salt_buf1[1]); + w[ 6] |= swap32_S (salt_buf1[2]); + w[ 7] |= swap32_S (salt_buf1[3]); + w[ 8] |= swap32_S (salt_buf2[0]); + w[ 9] |= swap32_S (salt_buf2[1]); + w[10] |= swap32_S (salt_buf2[2]); + w[11] |= swap32_S (salt_buf2[3]); + w[12] |= swap32_S (salt_buf3[0]); + w[13] |= swap32_S (salt_buf3[1]); + w[14] |= swap32_S (salt_buf3[2]); + w[15] |= swap32_S (salt_buf3[3]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + w[15] = pw_salt_len * 8; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01730s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01730_a3-pure.cl b/OpenCL/m01730_a3-pure.cl deleted file mode 100644 index c408f8105..000000000 --- a/OpenCL/m01730_a3-pure.cl +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector_utf16beN (&ctx, w, pw_len); - - sha512_update_vector (&ctx, s, salt_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - - -__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector (&ctx); - - sha512_update_vector_utf16beN (&ctx, w, pw_len); - - sha512_update_vector (&ctx, s, salt_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01730_a3.cl b/OpenCL/m01730_a3.cl index 67a3d98d3..c408f8105 100644 --- a/OpenCL/m01730_a3.cl +++ b/OpenCL/m01730_a3.cl @@ -11,195 +11,54 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -void m01730m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01730_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - switch_buffer_by_offset_le_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, pw_len); + u32x w[64] = { 0 }; - w[ 0] |= swap32_S (salt_buf0[0]); - w[ 1] |= swap32_S (salt_buf0[1]); - w[ 2] |= swap32_S (salt_buf0[2]); - w[ 3] |= swap32_S (salt_buf0[3]); - w[ 4] |= swap32_S (salt_buf1[0]); - w[ 5] |= swap32_S (salt_buf1[1]); - w[ 6] |= swap32_S (salt_buf1[2]); - w[ 7] |= swap32_S (salt_buf1[3]); - w[ 8] |= swap32_S (salt_buf2[0]); - w[ 9] |= swap32_S (salt_buf2[1]); - w[10] |= swap32_S (salt_buf2[2]); - w[11] |= swap32_S (salt_buf2[3]); - w[12] |= swap32_S (salt_buf3[0]); - w[13] |= swap32_S (salt_buf3[1]); - w[14] |= swap32_S (salt_buf3[2]); - w[15] |= swap32_S (salt_buf3[3]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); - w[15] = pw_salt_len * 8; + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -207,58 +66,38 @@ void m01730m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector_utf16beN (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_update_vector (&ctx, s, salt_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01730s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) + +__kernel void m01730_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -272,11 +111,41 @@ void m01730s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -284,274 +153,23 @@ void m01730s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha512_ctx_vector_t ctx; - u64x digest[8]; + sha512_init_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; + sha512_update_vector_utf16beN (&ctx, w, pw_len); - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha512_update_vector (&ctx, s, salt_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01730_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01730_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01730_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01730_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01730_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01730_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01730s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01740_a0-optimized.cl b/OpenCL/m01740_a0-optimized.cl new file mode 100644 index 000000000..d5366f7c6 --- /dev/null +++ b/OpenCL/m01740_a0-optimized.cl @@ -0,0 +1,468 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x out_len2 = out_len * 2; + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len2 + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01740_a0-pure.cl b/OpenCL/m01740_a0-pure.cl deleted file mode 100644 index d489d6e67..000000000 --- a/OpenCL/m01740_a0-pure.cl +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx = ctx0; - - sha512_update_utf16le_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_ctx_t ctx = ctx0; - - sha512_update_utf16le_swap (&ctx, w, pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01740_a0.cl b/OpenCL/m01740_a0.cl index d5366f7c6..d489d6e67 100644 --- a/OpenCL/m01740_a0.cl +++ b/OpenCL/m01740_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,348 +12,77 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha512_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha512_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -366,103 +95,48 @@ __kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_ctx_t ctx = ctx0; - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); + sha512_update_utf16le_swap (&ctx, w, pw_len); - const u32x out_len2 = out_len * 2; + sha512_final (&ctx); - /** - * prepend salt - */ + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - const u32x out_salt_len = out_len2 + salt_len; - - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01740_a1-optimized.cl b/OpenCL/m01740_a1-optimized.cl new file mode 100644 index 000000000..79fa26855 --- /dev/null +++ b/OpenCL/m01740_a1-optimized.cl @@ -0,0 +1,582 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + make_utf16le (w1, w2, w3); + make_utf16le (w0, w0, w1); + + const u32x pw_len2 = pw_len * 2; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len2 + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01740_a1-pure.cl b/OpenCL/m01740_a1-pure.cl deleted file mode 100644 index d29b54c1f..000000000 --- a/OpenCL/m01740_a1-pure.cl +++ /dev/null @@ -1,114 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha512_ctx_t ctx = ctx0; - - sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha512_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.h[7]); - const u32 r1 = h32_from_64_S (ctx.h[7]); - const u32 r2 = l32_from_64_S (ctx.h[3]); - const u32 r3 = h32_from_64_S (ctx.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01740_a1.cl b/OpenCL/m01740_a1.cl index 79fa26855..d29b54c1f 100644 --- a/OpenCL/m01740_a1.cl +++ b/OpenCL/m01740_a1.cl @@ -3,413 +3,71 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha512_ctx_t ctx0; - if (gid >= gid_max) return; + sha512_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -422,161 +80,35 @@ __kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + sha512_ctx_t ctx0; + + sha512_init (&ctx0); + + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha512_update_global_utf16le_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha512_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha512_update_global_utf16le_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha512_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = l32_from_64_S (ctx.h[7]); + const u32 r1 = h32_from_64_S (ctx.h[7]); + const u32 r2 = l32_from_64_S (ctx.h[3]); + const u32 r3 = h32_from_64_S (ctx.h[3]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - make_utf16le (w1, w2, w3); - make_utf16le (w0, w0, w1); - - const u32x pw_len2 = pw_len * 2; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len2 + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; - - u64x digest[8]; - - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01740_a3-optimized.cl b/OpenCL/m01740_a3-optimized.cl new file mode 100644 index 000000000..e78a02ae0 --- /dev/null +++ b/OpenCL/m01740_a3-optimized.cl @@ -0,0 +1,722 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = g; + digest[7] = h; +} + +void m01740m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01740s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = 0; + w3_t[3] = pw_salt_len * 8; + + u64x digest[8]; + + digest[0] = SHA512M_A; + digest[1] = SHA512M_B; + digest[2] = SHA512M_C; + digest[3] = SHA512M_D; + digest[4] = SHA512M_E; + digest[5] = SHA512M_F; + digest[6] = SHA512M_G; + digest[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01740_a3-pure.cl b/OpenCL/m01740_a3-pure.cl deleted file mode 100644 index 6ae09ef93..000000000 --- a/OpenCL/m01740_a3-pure.cl +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector_from_scalar (&ctx, &ctx0); - - sha512_update_vector_utf16beN (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_ctx_t ctx0; - - sha512_init (&ctx0); - - sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_ctx_vector_t ctx; - - sha512_init_vector_from_scalar (&ctx, &ctx0); - - sha512_update_vector_utf16beN (&ctx, w, pw_len); - - sha512_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[7]); - const u32x r1 = h32_from_64 (ctx.h[7]); - const u32x r2 = l32_from_64 (ctx.h[3]); - const u32x r3 = h32_from_64 (ctx.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01740_a3.cl b/OpenCL/m01740_a3.cl index e78a02ae0..6ae09ef93 100644 --- a/OpenCL/m01740_a3.cl +++ b/OpenCL/m01740_a3.cl @@ -11,284 +11,83 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = g; - digest[7] = h; -} - -void m01740m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01740_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha512_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha512_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha512 - */ + sha512_ctx_vector_t ctx; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; + sha512_update_vector_utf16beN (&ctx, w, pw_len); - u64x digest[8]; + sha512_final_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01740s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01740_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -303,420 +102,55 @@ void m01740s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + sha512_ctx_t ctx0; - switch_buffer_by_offset_be_S (w0, w1, w2, w3, salt_len); + sha512_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + sha512_update_global_swap (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - overwrite_at_be_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0; - /** - * sha512 - */ + sha512_ctx_vector_t ctx; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = 0; - w3_t[3] = pw_salt_len * 8; + sha512_update_vector_utf16beN (&ctx, w, pw_len); - u64x digest[8]; + sha512_final_vector (&ctx); - digest[0] = SHA512M_A; - digest[1] = SHA512M_B; - digest[2] = SHA512M_C; - digest[3] = SHA512M_D; - digest[4] = SHA512M_E; - digest[5] = SHA512M_F; - digest[6] = SHA512M_G; - digest[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.h[7]); + const u32x r1 = h32_from_64 (ctx.h[7]); + const u32x r2 = l32_from_64 (ctx.h[3]); + const u32x r3 = h32_from_64 (ctx.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01740_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01740_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01740_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01740_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01740_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01740_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01740s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01750_a0-optimized.cl b/OpenCL/m01750_a0-optimized.cl new file mode 100644 index 000000000..2a6d5bc25 --- /dev/null +++ b/OpenCL/m01750_a0-optimized.cl @@ -0,0 +1,521 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; + w2_t[0] = (u64x) 0x3636363636363636; + w2_t[1] = (u64x) 0x3636363636363636; + w2_t[2] = (u64x) 0x3636363636363636; + w2_t[3] = (u64x) 0x3636363636363636; + w3_t[0] = (u64x) 0x3636363636363636; + w3_t[1] = (u64x) 0x3636363636363636; + w3_t[2] = (u64x) 0x3636363636363636; + w3_t[3] = (u64x) 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + + /** + * pads + */ + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01750_a0-pure.cl b/OpenCL/m01750_a0-pure.cl deleted file mode 100644 index 0ff9c0346..000000000 --- a/OpenCL/m01750_a0-pure.cl +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_hmac_ctx_t ctx; - - sha512_hmac_init_swap (&ctx, w, pw_len); - - sha512_hmac_update (&ctx, s, salt_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_hmac_ctx_t ctx; - - sha512_hmac_init_swap (&ctx, w, pw_len); - - sha512_hmac_update (&ctx, s, salt_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01750_a0.cl b/OpenCL/m01750_a0.cl index 2a6d5bc25..0ff9c0346 100644 --- a/OpenCL/m01750_a0.cl +++ b/OpenCL/m01750_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,433 +12,86 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; - w2_t[0] = (u64x) 0x3636363636363636; - w2_t[1] = (u64x) 0x3636363636363636; - w2_t[2] = (u64x) 0x3636363636363636; - w2_t[3] = (u64x) 0x3636363636363636; - w3_t[0] = (u64x) 0x3636363636363636; - w3_t[1] = (u64x) 0x3636363636363636; - w3_t[2] = (u64x) 0x3636363636363636; - w3_t[3] = (u64x) 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha512_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha512_hmac_update (&ctx, s, salt_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_final (&ctx); - hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (128 + salt_len) * 8; - - u64x digest[8]; - - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -451,71 +104,57 @@ __kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_hmac_ctx_t ctx; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); + sha512_hmac_init_swap (&ctx, w, pw_len); - /** - * pads - */ + sha512_hmac_update (&ctx, s, salt_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_final (&ctx); - hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (128 + salt_len) * 8; - - u64x digest[8]; - - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01750_a1-optimized.cl b/OpenCL/m01750_a1-optimized.cl new file mode 100644 index 000000000..ce15a1534 --- /dev/null +++ b/OpenCL/m01750_a1-optimized.cl @@ -0,0 +1,655 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; + w2_t[0] = (u64x) 0x3636363636363636; + w2_t[1] = (u64x) 0x3636363636363636; + w2_t[2] = (u64x) 0x3636363636363636; + w2_t[3] = (u64x) 0x3636363636363636; + w3_t[0] = (u64x) 0x3636363636363636; + w3_t[1] = (u64x) 0x3636363636363636; + w3_t[2] = (u64x) 0x3636363636363636; + w3_t[3] = (u64x) 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = swap32 (w3[2]); + w3[3] = swap32 (w3[3]); + + /** + * pads + */ + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + w2[0] = salt_buf2[0]; + w2[1] = salt_buf2[1]; + w2[2] = salt_buf2[2]; + w2[3] = salt_buf2[3]; + w3[0] = salt_buf3[0]; + w3[1] = salt_buf3[1]; + w3[2] = 0; + w3[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01750_a1-pure.cl b/OpenCL/m01750_a1-pure.cl deleted file mode 100644 index 27be72d5e..000000000 --- a/OpenCL/m01750_a1-pure.cl +++ /dev/null @@ -1,198 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha512_hmac_ctx_vector_t ctx; - - sha512_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha512_hmac_update_vector (&ctx, s, salt_len); - - sha512_hmac_final_vector (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha512_hmac_ctx_vector_t ctx; - - sha512_hmac_init_vector (&ctx, c, pw_len + comb_len); - - sha512_hmac_update_vector (&ctx, s, salt_len); - - sha512_hmac_final_vector (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01750_a1.cl b/OpenCL/m01750_a1.cl index ce15a1534..27be72d5e 100644 --- a/OpenCL/m01750_a1.cl +++ b/OpenCL/m01750_a1.cl @@ -3,508 +3,113 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; - w2_t[0] = (u64x) 0x3636363636363636; - w2_t[1] = (u64x) 0x3636363636363636; - w2_t[2] = (u64x) 0x3636363636363636; - w2_t[3] = (u64x) 0x3636363636363636; - w3_t[0] = (u64x) 0x3636363636363636; - w3_t[1] = (u64x) 0x3636363636363636; - w3_t[2] = (u64x) 0x3636363636363636; - w3_t[3] = (u64x) 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha512_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha512_hmac_init_vector (&ctx, c, pw_len + comb_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_update_vector (&ctx, s, salt_len); - hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + sha512_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (128 + salt_len) * 8; + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - u64x digest[8]; - - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -517,139 +122,77 @@ __kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = swap32 (w3[2]); - w3[3] = swap32 (w3[3]); + sha512_hmac_ctx_vector_t ctx; - /** - * pads - */ + sha512_hmac_init_vector (&ctx, c, pw_len + comb_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_update_vector (&ctx, s, salt_len); - hmac_sha512_pad (w0, w1, w2, w3, ipad, opad); + sha512_hmac_final_vector (&ctx); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - w2[0] = salt_buf2[0]; - w2[1] = salt_buf2[1]; - w2[2] = salt_buf2[2]; - w2[3] = salt_buf2[3]; - w3[0] = salt_buf3[0]; - w3[1] = salt_buf3[1]; - w3[2] = 0; - w3[3] = (128 + salt_len) * 8; + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - u64x digest[8]; - - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01750_a3-optimized.cl b/OpenCL/m01750_a3-optimized.cl new file mode 100644 index 000000000..2f0a9fa52 --- /dev/null +++ b/OpenCL/m01750_a3-optimized.cl @@ -0,0 +1,767 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ 0x3636363636363636; + w2_t[0] = 0x3636363636363636; + w2_t[1] = 0x3636363636363636; + w2_t[2] = 0x3636363636363636; + w2_t[3] = 0x3636363636363636; + w3_t[0] = 0x3636363636363636; + w3_t[1] = 0x3636363636363636; + w3_t[2] = 0x3636363636363636; + w3_t[3] = 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ 0x5c5c5c5c5c5c5c5c; + w2_t[0] = 0x5c5c5c5c5c5c5c5c; + w2_t[1] = 0x5c5c5c5c5c5c5c5c; + w2_t[2] = 0x5c5c5c5c5c5c5c5c; + w2_t[3] = 0x5c5c5c5c5c5c5c5c; + w3_t[0] = 0x5c5c5c5c5c5c5c5c; + w3_t[1] = 0x5c5c5c5c5c5c5c5c; + w3_t[2] = 0x5c5c5c5c5c5c5c5c; + w3_t[3] = 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +void m01750m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01750s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = 0; + w3_t[3] = (128 + salt_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01750_a3-pure.cl b/OpenCL/m01750_a3-pure.cl deleted file mode 100644 index 28c6a4b38..000000000 --- a/OpenCL/m01750_a3-pure.cl +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_hmac_ctx_vector_t ctx; - - sha512_hmac_init_vector (&ctx, w, pw_len); - - sha512_hmac_update_vector (&ctx, s, salt_len); - - sha512_hmac_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.opad.h[7]); - const u32x r1 = h32_from_64 (ctx.opad.h[7]); - const u32x r2 = l32_from_64 (ctx.opad.h[3]); - const u32x r3 = h32_from_64 (ctx.opad.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_hmac_ctx_vector_t ctx; - - sha512_hmac_init_vector (&ctx, w, pw_len); - - sha512_hmac_update_vector (&ctx, s, salt_len); - - sha512_hmac_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.opad.h[7]); - const u32x r1 = h32_from_64 (ctx.opad.h[7]); - const u32x r2 = l32_from_64 (ctx.opad.h[3]); - const u32x r3 = h32_from_64 (ctx.opad.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01750_a3.cl b/OpenCL/m01750_a3.cl index 2f0a9fa52..28c6a4b38 100644 --- a/OpenCL/m01750_a3.cl +++ b/OpenCL/m01750_a3.cl @@ -11,393 +11,90 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ 0x3636363636363636; - w2_t[0] = 0x3636363636363636; - w2_t[1] = 0x3636363636363636; - w2_t[2] = 0x3636363636363636; - w2_t[3] = 0x3636363636363636; - w3_t[0] = 0x3636363636363636; - w3_t[1] = 0x3636363636363636; - w3_t[2] = 0x3636363636363636; - w3_t[3] = 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ 0x5c5c5c5c5c5c5c5c; - w2_t[0] = 0x5c5c5c5c5c5c5c5c; - w2_t[1] = 0x5c5c5c5c5c5c5c5c; - w2_t[2] = 0x5c5c5c5c5c5c5c5c; - w2_t[3] = 0x5c5c5c5c5c5c5c5c; - w3_t[0] = 0x5c5c5c5c5c5c5c5c; - w3_t[1] = 0x5c5c5c5c5c5c5c5c; - w3_t[2] = 0x5c5c5c5c5c5c5c5c; - w3_t[3] = 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -void m01750m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01750_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha512_hmac_init_vector (&ctx, w, pw_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_update_vector (&ctx, s, salt_len); - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha512_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (128 + salt_len) * 8; - - u64x digest[8]; - - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.opad.h[7]); + const u32x r1 = h32_from_64 (ctx.opad.h[7]); + const u32x r2 = l32_from_64 (ctx.opad.h[3]); + const u32x r3 = h32_from_64 (ctx.opad.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01750s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01750_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; + if (gid >= gid_max) return; /** * digest @@ -411,357 +108,63 @@ void m01750s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - /** - * pads - */ + w[0] = w0; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha512_hmac_ctx_vector_t ctx; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + sha512_hmac_init_vector (&ctx, w, pw_len); - u64x ipad[8]; - u64x opad[8]; + sha512_hmac_update_vector (&ctx, s, salt_len); - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha512_hmac_final_vector (&ctx); - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = 0; - w3_t[3] = (128 + salt_len) * 8; - - u64x digest[8]; - - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + const u32x r0 = l32_from_64 (ctx.opad.h[7]); + const u32x r1 = h32_from_64 (ctx.opad.h[7]); + const u32x r2 = l32_from_64 (ctx.opad.h[3]); + const u32x r3 = h32_from_64 (ctx.opad.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01750_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01750_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01750_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01750_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01750_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01750_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01750s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01760_a0-optimized.cl b/OpenCL/m01760_a0-optimized.cl new file mode 100644 index 000000000..fed50da2a --- /dev/null +++ b/OpenCL/m01760_a0-optimized.cl @@ -0,0 +1,551 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; + w2_t[0] = (u64x) 0x3636363636363636; + w2_t[1] = (u64x) 0x3636363636363636; + w2_t[2] = (u64x) 0x3636363636363636; + w2_t[3] = (u64x) 0x3636363636363636; + w3_t[0] = (u64x) 0x3636363636363636; + w3_t[1] = (u64x) 0x3636363636363636; + w3_t[2] = (u64x) 0x3636363636363636; + w3_t[3] = (u64x) 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + out_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + out_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01760_a0-pure.cl b/OpenCL/m01760_a0-pure.cl deleted file mode 100644 index f977db7ac..000000000 --- a/OpenCL/m01760_a0-pure.cl +++ /dev/null @@ -1,164 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_t ctx0; - - sha512_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_hmac_ctx_t ctx = ctx0; - - sha512_hmac_update_swap (&ctx, w, pw_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_t ctx0; - - sha512_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha512_hmac_ctx_t ctx = ctx0; - - sha512_hmac_update_swap (&ctx, w, pw_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01760_a0.cl b/OpenCL/m01760_a0.cl index fed50da2a..f977db7ac 100644 --- a/OpenCL/m01760_a0.cl +++ b/OpenCL/m01760_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,479 +12,88 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; - w2_t[0] = (u64x) 0x3636363636363636; - w2_t[1] = (u64x) 0x3636363636363636; - w2_t[2] = (u64x) 0x3636363636363636; - w2_t[3] = (u64x) 0x3636363636363636; - w3_t[0] = (u64x) 0x3636363636363636; - w3_t[1] = (u64x) 0x3636363636363636; - w3_t[2] = (u64x) 0x3636363636363636; - w3_t[3] = (u64x) 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u64x ipad[8]; - u64x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha512_hmac_ctx_t ctx0; + + sha512_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha512_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + out_len) * 8; + sha512_hmac_final (&ctx); - u64x digest[8]; + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u64x ipad[8]; - u64x opad[8]; - - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -497,55 +106,59 @@ __kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_hmac_ctx_t ctx0; + + sha512_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha512_hmac_ctx_t ctx = ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + sha512_hmac_update_swap (&ctx, w, pw_len); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + out_len) * 8; + sha512_hmac_final (&ctx); - u64x digest[8]; + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); - - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01760_a1-optimized.cl b/OpenCL/m01760_a1-optimized.cl new file mode 100644 index 000000000..d6f4b4f6d --- /dev/null +++ b/OpenCL/m01760_a1-optimized.cl @@ -0,0 +1,669 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; + w2_t[0] = (u64x) 0x3636363636363636; + w2_t[1] = (u64x) 0x3636363636363636; + w2_t[2] = (u64x) 0x3636363636363636; + w2_t[3] = (u64x) 0x3636363636363636; + w3_t[0] = (u64x) 0x3636363636363636; + w3_t[1] = (u64x) 0x3636363636363636; + w3_t[2] = (u64x) 0x3636363636363636; + w3_t[3] = (u64x) 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (128 + pw_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + + w0[0] = swap32 (w0[0]); + w0[1] = swap32 (w0[1]); + w0[2] = swap32 (w0[2]); + w0[3] = swap32 (w0[3]); + w1[0] = swap32 (w1[0]); + w1[1] = swap32 (w1[1]); + w1[2] = swap32 (w1[2]); + w1[3] = swap32 (w1[3]); + w2[0] = swap32 (w2[0]); + w2[1] = swap32 (w2[1]); + w2[2] = swap32 (w2[2]); + w2[3] = swap32 (w2[3]); + w3[0] = swap32 (w3[0]); + w3[1] = swap32 (w3[1]); + w3[2] = 0; + w3[3] = (128 + pw_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m01760_a1-pure.cl b/OpenCL/m01760_a1-pure.cl deleted file mode 100644 index 5403bcc5d..000000000 --- a/OpenCL/m01760_a1-pure.cl +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_t ctx0; - - sha512_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha512_hmac_ctx_t ctx = ctx0; - - sha512_hmac_update (&ctx, c, pw_len + comb_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_t ctx0; - - sha512_hmac_init (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - const u32 comb_len = combs_buf[il_pos].pw_len; - - u32 c[64]; - - #ifdef _unroll - #pragma unroll - #endif - for (int idx = 0; idx < 64; idx++) - { - c[idx] = swap32_S (combs_buf[il_pos].i[idx]); - } - - switch_buffer_by_offset_1x64_be_S (c, pw_len); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) - { - c[i] |= w[i]; - } - - sha512_hmac_ctx_t ctx = ctx0; - - sha512_hmac_update (&ctx, c, pw_len + comb_len); - - sha512_hmac_final (&ctx); - - const u32 r0 = l32_from_64_S (ctx.opad.h[7]); - const u32 r1 = h32_from_64_S (ctx.opad.h[7]); - const u32 r2 = l32_from_64_S (ctx.opad.h[3]); - const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01760_a1.cl b/OpenCL/m01760_a1.cl index d6f4b4f6d..5403bcc5d 100644 --- a/OpenCL/m01760_a1.cl +++ b/OpenCL/m01760_a1.cl @@ -3,546 +3,115 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; - w2_t[0] = (u64x) 0x3636363636363636; - w2_t[1] = (u64x) 0x3636363636363636; - w2_t[2] = (u64x) 0x3636363636363636; - w2_t[3] = (u64x) 0x3636363636363636; - w3_t[0] = (u64x) 0x3636363636363636; - w3_t[1] = (u64x) 0x3636363636363636; - w3_t[2] = (u64x) 0x3636363636363636; - w3_t[3] = (u64x) 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + const u32 pw_len = pws[gid].pw_len; - if (gid >= gid_max) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 pw_buf0[4]; - u32 pw_buf1[4]; + u32 w[64] = { 0 }; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32 s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u64x ipad[8]; - u64x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha512_hmac_ctx_t ctx0; + + sha512_hmac_init (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha512_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (128 + pw_len) * 8; + sha512_hmac_update (&ctx, c, pw_len + comb_len); - u64x digest[8]; + sha512_hmac_final (&ctx); - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u64x ipad[8]; - u64x opad[8]; - - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); - /** * digest */ @@ -555,115 +124,79 @@ __kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_hmac_ctx_t ctx0; + + sha512_hmac_init (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + const u32 comb_len = combs_buf[il_pos].pw_len; - const u32x pw_len = pw_l_len + pw_r_len; + u32 c[64]; - /** - * concat password candidate - */ - - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; - - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + #ifdef _unroll + #pragma unroll + #endif + for (int idx = 0; idx < 64; idx++) { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + c[idx] = swap32_S (combs_buf[il_pos].i[idx]); } - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + switch_buffer_by_offset_1x64_be_S (c, pw_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) + { + c[i] |= w[i]; + } - append_0x80_4x4_VV (w0, w1, w2, w3, pw_len); + sha512_hmac_ctx_t ctx = ctx0; - w0[0] = swap32 (w0[0]); - w0[1] = swap32 (w0[1]); - w0[2] = swap32 (w0[2]); - w0[3] = swap32 (w0[3]); - w1[0] = swap32 (w1[0]); - w1[1] = swap32 (w1[1]); - w1[2] = swap32 (w1[2]); - w1[3] = swap32 (w1[3]); - w2[0] = swap32 (w2[0]); - w2[1] = swap32 (w2[1]); - w2[2] = swap32 (w2[2]); - w2[3] = swap32 (w2[3]); - w3[0] = swap32 (w3[0]); - w3[1] = swap32 (w3[1]); - w3[2] = 0; - w3[3] = (128 + pw_len) * 8; + sha512_hmac_update (&ctx, c, pw_len + comb_len); - u64x digest[8]; + sha512_hmac_final (&ctx); - hmac_sha512_run (w0, w1, w2, w3, ipad, opad, digest); + const u32 r0 = l32_from_64_S (ctx.opad.h[7]); + const u32 r1 = h32_from_64_S (ctx.opad.h[7]); + const u32 r2 = l32_from_64_S (ctx.opad.h[3]); + const u32 r3 = h32_from_64_S (ctx.opad.h[3]); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m01760_a3-optimized.cl b/OpenCL/m01760_a3-optimized.cl new file mode 100644 index 000000000..c9a79fb53 --- /dev/null +++ b/OpenCL/m01760_a3-optimized.cl @@ -0,0 +1,767 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) +{ + u64x w0_t = w0[0]; + u64x w1_t = w0[1]; + u64x w2_t = w0[2]; + u64x w3_t = w0[3]; + u64x w4_t = w1[0]; + u64x w5_t = w1[1]; + u64x w6_t = w1[2]; + u64x w7_t = w1[3]; + u64x w8_t = w2[0]; + u64x w9_t = w2[1]; + u64x wa_t = w2[2]; + u64x wb_t = w2[3]; + u64x wc_t = w3[0]; + u64x wd_t = w3[1]; + u64x we_t = w3[2]; + u64x wf_t = w3[3]; + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; + w2_t[0] = (u64x) 0x3636363636363636; + w2_t[1] = (u64x) 0x3636363636363636; + w2_t[2] = (u64x) 0x3636363636363636; + w2_t[3] = (u64x) 0x3636363636363636; + w3_t[0] = (u64x) 0x3636363636363636; + w3_t[1] = (u64x) 0x3636363636363636; + w3_t[2] = (u64x) 0x3636363636363636; + w3_t[3] = (u64x) 0x3636363636363636; + + ipad[0] = SHA512M_A; + ipad[1] = SHA512M_B; + ipad[2] = SHA512M_C; + ipad[3] = SHA512M_D; + ipad[4] = SHA512M_E; + ipad[5] = SHA512M_F; + ipad[6] = SHA512M_G; + ipad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); + + w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; + w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; + + opad[0] = SHA512M_A; + opad[1] = SHA512M_B; + opad[2] = SHA512M_C; + opad[3] = SHA512M_D; + opad[4] = SHA512M_E; + opad[5] = SHA512M_F; + opad[6] = SHA512M_G; + opad[7] = SHA512M_H; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); +} + +void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) +{ + u64x w0_t[4]; + u64x w1_t[4]; + u64x w2_t[4]; + u64x w3_t[4]; + + w0_t[0] = hl32_to_64 (w0[0], w0[1]); + w0_t[1] = hl32_to_64 (w0[2], w0[3]); + w0_t[2] = hl32_to_64 (w1[0], w1[1]); + w0_t[3] = hl32_to_64 (w1[2], w1[3]); + w1_t[0] = hl32_to_64 (w2[0], w2[1]); + w1_t[1] = hl32_to_64 (w2[2], w2[3]); + w1_t[2] = hl32_to_64 (w3[0], w3[1]); + w1_t[3] = 0; + w2_t[0] = 0; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = hl32_to_64 (w3[2], w3[3]); + + digest[0] = ipad[0]; + digest[1] = ipad[1]; + digest[2] = ipad[2]; + digest[3] = ipad[3]; + digest[4] = ipad[4]; + digest[5] = ipad[5]; + digest[6] = ipad[6]; + digest[7] = ipad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); + + w0_t[0] = digest[0]; + w0_t[1] = digest[1]; + w0_t[2] = digest[2]; + w0_t[3] = digest[3]; + w1_t[0] = digest[4]; + w1_t[1] = digest[5]; + w1_t[2] = digest[6]; + w1_t[3] = digest[7]; + w2_t[0] = 0x8000000000000000; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = (128 + 64) * 8; + + digest[0] = opad[0]; + digest[1] = opad[1]; + digest[2] = opad[2]; + digest[3] = opad[3]; + digest[4] = opad[4]; + digest[5] = opad[5]; + digest[6] = opad[6]; + digest[7] = opad[7]; + + sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); +} + +void m01760m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (128 + pw_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m01760s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); + salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); + salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); + salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); + salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); + salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); + salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); + salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); + salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); + salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); + salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); + salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); + salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); + salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); + salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); + salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * pads + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + w2_t[0] = salt_buf2[0]; + w2_t[1] = salt_buf2[1]; + w2_t[2] = salt_buf2[2]; + w2_t[3] = salt_buf2[3]; + w3_t[0] = salt_buf3[0]; + w3_t[1] = salt_buf3[1]; + w3_t[2] = salt_buf3[2]; + w3_t[3] = salt_buf3[3]; + + u64x ipad[8]; + u64x opad[8]; + + hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = 0; + w3_t[3] = (128 + pw_len) * 8; + + u64x digest[8]; + + hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + + const u32x r0 = l32_from_64 (digest[7]); + const u32x r1 = h32_from_64 (digest[7]); + const u32x r2 = l32_from_64 (digest[3]); + const u32x r3 = h32_from_64 (digest[3]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m01760_a3-pure.cl b/OpenCL/m01760_a3-pure.cl deleted file mode 100644 index f3b80a484..000000000 --- a/OpenCL/m01760_a3-pure.cl +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha512.cl" - -__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_vector_t ctx0; - - sha512_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_hmac_ctx_vector_t ctx = ctx0; - - sha512_hmac_update_vector (&ctx, w, pw_len); - - sha512_hmac_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.opad.h[7]); - const u32x r1 = h32_from_64 (ctx.opad.h[7]); - const u32x r2 = l32_from_64 (ctx.opad.h[3]); - const u32x r3 = h32_from_64 (ctx.opad.h[3]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - sha512_hmac_ctx_vector_t ctx0; - - sha512_hmac_init_vector (&ctx0, s, salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha512_hmac_ctx_vector_t ctx = ctx0; - - sha512_hmac_update_vector (&ctx, w, pw_len); - - sha512_hmac_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.opad.h[7]); - const u32x r1 = h32_from_64 (ctx.opad.h[7]); - const u32x r2 = l32_from_64 (ctx.opad.h[3]); - const u32x r3 = h32_from_64 (ctx.opad.h[3]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m01760_a3.cl b/OpenCL/m01760_a3.cl index c9a79fb53..f3b80a484 100644 --- a/OpenCL/m01760_a3.cl +++ b/OpenCL/m01760_a3.cl @@ -11,424 +11,92 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha512.cl" -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64x w0[4], const u64x w1[4], const u64x w2[4], const u64x w3[4], u64x digest[8]) -{ - u64x w0_t = w0[0]; - u64x w1_t = w0[1]; - u64x w2_t = w0[2]; - u64x w3_t = w0[3]; - u64x w4_t = w1[0]; - u64x w5_t = w1[1]; - u64x w6_t = w1[2]; - u64x w7_t = w1[3]; - u64x w8_t = w2[0]; - u64x w9_t = w2[1]; - u64x wa_t = w2[2]; - u64x wb_t = w2[3]; - u64x wc_t = w3[0]; - u64x wd_t = w3[1]; - u64x we_t = w3[2]; - u64x wf_t = w3[3]; - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void hmac_sha512_pad (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x3636363636363636; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x3636363636363636; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x3636363636363636; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x3636363636363636; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x3636363636363636; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x3636363636363636; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x3636363636363636; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x3636363636363636; - w2_t[0] = (u64x) 0x3636363636363636; - w2_t[1] = (u64x) 0x3636363636363636; - w2_t[2] = (u64x) 0x3636363636363636; - w2_t[3] = (u64x) 0x3636363636363636; - w3_t[0] = (u64x) 0x3636363636363636; - w3_t[1] = (u64x) 0x3636363636363636; - w3_t[2] = (u64x) 0x3636363636363636; - w3_t[3] = (u64x) 0x3636363636363636; - - ipad[0] = SHA512M_A; - ipad[1] = SHA512M_B; - ipad[2] = SHA512M_C; - ipad[3] = SHA512M_D; - ipad[4] = SHA512M_E; - ipad[5] = SHA512M_F; - ipad[6] = SHA512M_G; - ipad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, ipad); - - w0_t[0] = hl32_to_64 (w0[0], w0[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[1] = hl32_to_64 (w0[2], w0[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[2] = hl32_to_64 (w1[0], w1[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w0_t[3] = hl32_to_64 (w1[2], w1[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[0] = hl32_to_64 (w2[0], w2[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[1] = hl32_to_64 (w2[2], w2[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[2] = hl32_to_64 (w3[0], w3[1]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w1_t[3] = hl32_to_64 (w3[2], w3[3]) ^ (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w2_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[0] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[1] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[2] = (u64x) 0x5c5c5c5c5c5c5c5c; - w3_t[3] = (u64x) 0x5c5c5c5c5c5c5c5c; - - opad[0] = SHA512M_A; - opad[1] = SHA512M_B; - opad[2] = SHA512M_C; - opad[3] = SHA512M_D; - opad[4] = SHA512M_E; - opad[5] = SHA512M_F; - opad[6] = SHA512M_G; - opad[7] = SHA512M_H; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, opad); -} - -void hmac_sha512_run (u32x w0[4], u32x w1[4], u32x w2[4], u32x w3[4], u64x ipad[8], u64x opad[8], u64x digest[8]) -{ - u64x w0_t[4]; - u64x w1_t[4]; - u64x w2_t[4]; - u64x w3_t[4]; - - w0_t[0] = hl32_to_64 (w0[0], w0[1]); - w0_t[1] = hl32_to_64 (w0[2], w0[3]); - w0_t[2] = hl32_to_64 (w1[0], w1[1]); - w0_t[3] = hl32_to_64 (w1[2], w1[3]); - w1_t[0] = hl32_to_64 (w2[0], w2[1]); - w1_t[1] = hl32_to_64 (w2[2], w2[3]); - w1_t[2] = hl32_to_64 (w3[0], w3[1]); - w1_t[3] = 0; - w2_t[0] = 0; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = hl32_to_64 (w3[2], w3[3]); - - digest[0] = ipad[0]; - digest[1] = ipad[1]; - digest[2] = ipad[2]; - digest[3] = ipad[3]; - digest[4] = ipad[4]; - digest[5] = ipad[5]; - digest[6] = ipad[6]; - digest[7] = ipad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); - - w0_t[0] = digest[0]; - w0_t[1] = digest[1]; - w0_t[2] = digest[2]; - w0_t[3] = digest[3]; - w1_t[0] = digest[4]; - w1_t[1] = digest[5]; - w1_t[2] = digest[6]; - w1_t[3] = digest[7]; - w2_t[0] = 0x8000000000000000; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = (128 + 64) * 8; - - digest[0] = opad[0]; - digest[1] = opad[1]; - digest[2] = opad[2]; - digest[3] = opad[3]; - digest[4] = opad[4]; - digest[5] = opad[5]; - digest[6] = opad[6]; - digest[7] = opad[7]; - - sha512_transform (w0_t, w1_t, w2_t, w3_t, digest); -} - -void m01760m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esal_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01760_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** - * pads - */ + const u32 salt_lenv = ceil ((float) salt_len / 4); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + u32x s[64] = { 0 }; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); - u64x ipad[8]; - u64x opad[8]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + sha512_hmac_ctx_vector_t ctx0; + + sha512_hmac_init_vector (&ctx0, s, salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (128 + pw_len) * 8; + w[0] = w0; - u64x digest[8]; + sha512_hmac_ctx_vector_t ctx = ctx0; - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha512_hmac_update_vector (&ctx, w, pw_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_hmac_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.opad.h[7]); + const u32x r1 = h32_from_64 (ctx.opad.h[7]); + const u32x r2 = l32_from_64 (ctx.opad.h[3]); + const u32x r3 = h32_from_64 (ctx.opad.h[3]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m01760s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m01760_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 0]); - salt_buf0[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 1]); - salt_buf0[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 2]); - salt_buf0[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 3]); - salt_buf1[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 4]); - salt_buf1[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 5]); - salt_buf1[2] = swap32_S (salt_bufs[salt_pos].salt_buf[ 6]); - salt_buf1[3] = swap32_S (salt_bufs[salt_pos].salt_buf[ 7]); - salt_buf2[0] = swap32_S (salt_bufs[salt_pos].salt_buf[ 8]); - salt_buf2[1] = swap32_S (salt_bufs[salt_pos].salt_buf[ 9]); - salt_buf2[2] = swap32_S (salt_bufs[salt_pos].salt_buf[10]); - salt_buf2[3] = swap32_S (salt_bufs[salt_pos].salt_buf[11]); - salt_buf3[0] = swap32_S (salt_bufs[salt_pos].salt_buf[12]); - salt_buf3[1] = swap32_S (salt_bufs[salt_pos].salt_buf[13]); - salt_buf3[2] = swap32_S (salt_bufs[salt_pos].salt_buf[14]); - salt_buf3[3] = swap32_S (salt_bufs[salt_pos].salt_buf[15]); - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * pads - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; - w2_t[0] = salt_buf2[0]; - w2_t[1] = salt_buf2[1]; - w2_t[2] = salt_buf2[2]; - w2_t[3] = salt_buf2[3]; - w3_t[0] = salt_buf3[0]; - w3_t[1] = salt_buf3[1]; - w3_t[2] = salt_buf3[2]; - w3_t[3] = salt_buf3[3]; - - u64x ipad[8]; - u64x opad[8]; - - hmac_sha512_pad (w0_t, w1_t, w2_t, w3_t, ipad, opad); + if (gid >= gid_max) return; /** * digest @@ -442,326 +110,65 @@ void m01760s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + sha512_hmac_ctx_vector_t ctx0; + + sha512_hmac_init_vector (&ctx0, s, salt_len); + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = 0; - w3_t[3] = (128 + pw_len) * 8; + w[0] = w0; - u64x digest[8]; + sha512_hmac_ctx_vector_t ctx = ctx0; - hmac_sha512_run (w0_t, w1_t, w2_t, w3_t, ipad, opad, digest); + sha512_hmac_update_vector (&ctx, w, pw_len); - const u32x r0 = l32_from_64 (digest[7]); - const u32x r1 = h32_from_64 (digest[7]); - const u32x r2 = l32_from_64 (digest[3]); - const u32x r3 = h32_from_64 (digest[3]); + sha512_hmac_final_vector (&ctx); + + const u32x r0 = l32_from_64 (ctx.opad.h[7]); + const u32x r1 = h32_from_64 (ctx.opad.h[7]); + const u32x r2 = l32_from_64 (ctx.opad.h[3]); + const u32x r3 = h32_from_64 (ctx.opad.h[3]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m01760_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01760_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01760_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01760_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01760_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m01760_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m01760s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m01800-optimized.cl b/OpenCL/m01800-optimized.cl new file mode 100644 index 000000000..8ea852c61 --- /dev/null +++ b/OpenCL/m01800-optimized.cl @@ -0,0 +1,561 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +// Buggy drivers... + +#ifdef IS_AMD +#define STATE_DECL volatile +//#define STATE_DECL +#else +#define STATE_DECL +#endif + +#define PUTCHAR64_BE(a,p,c) ((u8 *)(a))[(p) ^ 7] = (u8) (c) +#define GETCHAR64_BE(a,p) ((u8 *)(a))[(p) ^ 7] + +typedef struct +{ + u64 state[8]; + u64 buf[16]; + int len; + +} sha512_ctx_t; + +__constant u64a k_sha512[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha512_transform (const u64 *w, u64 *digest) +{ + u64 w0_t = w[ 0]; + u64 w1_t = w[ 1]; + u64 w2_t = w[ 2]; + u64 w3_t = w[ 3]; + u64 w4_t = w[ 4]; + u64 w5_t = w[ 5]; + u64 w6_t = w[ 6]; + u64 w7_t = w[ 7]; + u64 w8_t = w[ 8]; + u64 w9_t = w[ 9]; + u64 wa_t = w[10]; + u64 wb_t = w[11]; + u64 wc_t = w[12]; + u64 wd_t = w[13]; + u64 we_t = w[14]; + u64 wf_t = w[15]; + + STATE_DECL u64 a = digest[0]; + STATE_DECL u64 b = digest[1]; + STATE_DECL u64 c = digest[2]; + STATE_DECL u64 d = digest[3]; + STATE_DECL u64 e = digest[4]; + STATE_DECL u64 f = digest[5]; + STATE_DECL u64 g = digest[6]; + STATE_DECL u64 h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void sha512_init (sha512_ctx_t *sha512_ctx) +{ + sha512_ctx->state[0] = SHA512M_A; + sha512_ctx->state[1] = SHA512M_B; + sha512_ctx->state[2] = SHA512M_C; + sha512_ctx->state[3] = SHA512M_D; + sha512_ctx->state[4] = SHA512M_E; + sha512_ctx->state[5] = SHA512M_F; + sha512_ctx->state[6] = SHA512M_G; + sha512_ctx->state[7] = SHA512M_H; + + sha512_ctx->len = 0; +} + +void sha512_update (sha512_ctx_t *sha512_ctx, const u64 *buf, int len) +{ + int pos = sha512_ctx->len & 0x7f; + + sha512_ctx->len += len; + + if ((pos + len) < 128) + { + for (int i = 0; i < len; i++) + { + PUTCHAR64_BE (sha512_ctx->buf, pos++, GETCHAR64_BE (buf, i)); + } + + return; + } + + int cnt = 128 - pos; + + for (int i = 0; i < cnt; i++) + { + PUTCHAR64_BE (sha512_ctx->buf, pos++, GETCHAR64_BE (buf, i)); + } + + sha512_transform (sha512_ctx->buf, sha512_ctx->state); + + len -= cnt; + + for (int i = 0; i < len; i++) + { + PUTCHAR64_BE (sha512_ctx->buf, i, GETCHAR64_BE (buf, cnt + i)); + } +} + +void sha512_final (sha512_ctx_t *sha512_ctx) +{ + int pos = sha512_ctx->len & 0x7f; + + for (int i = pos; i < 128; i++) + { + PUTCHAR64_BE (sha512_ctx->buf, i, 0); + } + + PUTCHAR64_BE (sha512_ctx->buf, pos, 0x80); + + if (pos >= 112) + { + sha512_transform (sha512_ctx->buf, sha512_ctx->state); + + sha512_ctx->buf[ 0] = 0; + sha512_ctx->buf[ 1] = 0; + sha512_ctx->buf[ 2] = 0; + sha512_ctx->buf[ 3] = 0; + sha512_ctx->buf[ 4] = 0; + sha512_ctx->buf[ 5] = 0; + sha512_ctx->buf[ 6] = 0; + sha512_ctx->buf[ 7] = 0; + sha512_ctx->buf[ 8] = 0; + sha512_ctx->buf[ 9] = 0; + sha512_ctx->buf[10] = 0; + sha512_ctx->buf[11] = 0; + sha512_ctx->buf[12] = 0; + sha512_ctx->buf[13] = 0; + sha512_ctx->buf[14] = 0; + sha512_ctx->buf[15] = 0; + } + + sha512_ctx->buf[15] = sha512_ctx->len * 8; + + sha512_transform (sha512_ctx->buf, sha512_ctx->state); +} + +__kernel void m01800_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[4]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; + + u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * buffers + */ + + u64 pw[2]; + + pw[0] = swap64 (hl32_to_64 (w0[1], w0[0])); + pw[1] = swap64 (hl32_to_64 (w0[3], w0[2])); + + u64 salt[2]; + + salt[0] = swap64 (hl32_to_64 (salt_buf[1], salt_buf[0])); + salt[1] = swap64 (hl32_to_64 (salt_buf[3], salt_buf[2])); + + /** + * begin + */ + + sha512_ctx_t sha512_ctx; + + sha512_init (&sha512_ctx); + + sha512_update (&sha512_ctx, pw, pw_len); + sha512_update (&sha512_ctx, salt, salt_len); + sha512_update (&sha512_ctx, pw, pw_len); + + sha512_final (&sha512_ctx); + + u64 tmp[8]; + + tmp[0] = sha512_ctx.state[0]; + tmp[1] = sha512_ctx.state[1]; + tmp[2] = sha512_ctx.state[2]; + tmp[3] = sha512_ctx.state[3]; + tmp[4] = sha512_ctx.state[4]; + tmp[5] = sha512_ctx.state[5]; + tmp[6] = sha512_ctx.state[6]; + tmp[7] = sha512_ctx.state[7]; + + sha512_init (&sha512_ctx); + + sha512_update (&sha512_ctx, pw, pw_len); + sha512_update (&sha512_ctx, salt, salt_len); + sha512_update (&sha512_ctx, tmp, pw_len); + + for (u32 j = pw_len; j; j >>= 1) + { + if (j & 1) + { + sha512_update (&sha512_ctx, tmp, 64); + } + else + { + sha512_update (&sha512_ctx, pw, pw_len); + } + } + + sha512_final (&sha512_ctx); + + tmps[gid].l_alt_result[0] = sha512_ctx.state[0]; + tmps[gid].l_alt_result[1] = sha512_ctx.state[1]; + tmps[gid].l_alt_result[2] = sha512_ctx.state[2]; + tmps[gid].l_alt_result[3] = sha512_ctx.state[3]; + tmps[gid].l_alt_result[4] = sha512_ctx.state[4]; + tmps[gid].l_alt_result[5] = sha512_ctx.state[5]; + tmps[gid].l_alt_result[6] = sha512_ctx.state[6]; + tmps[gid].l_alt_result[7] = sha512_ctx.state[7]; + + // p_bytes + + sha512_init (&sha512_ctx); + + for (u32 j = 0; j < pw_len; j++) + { + sha512_update (&sha512_ctx, pw, pw_len); + } + + sha512_final (&sha512_ctx); + + tmps[gid].l_p_bytes[0] = sha512_ctx.state[0]; + tmps[gid].l_p_bytes[1] = sha512_ctx.state[1]; + + // s_bytes + + sha512_init (&sha512_ctx); + + for (u32 j = 0; j < 16 + ((tmps[gid].l_alt_result[0] >> 56) & 0xff); j++) + { + sha512_update (&sha512_ctx, salt, salt_len); + } + + sha512_final (&sha512_ctx); + + tmps[gid].l_s_bytes[0] = sha512_ctx.state[0]; + tmps[gid].l_s_bytes[1] = sha512_ctx.state[1]; +} + +__kernel void m01800_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u64 l_p_bytes0[2]; + + l_p_bytes0[0] = tmps[gid].l_p_bytes[0]; + l_p_bytes0[1] = tmps[gid].l_p_bytes[1]; + + const u32 pw_len = pws[gid].pw_len; + + u64 l_s_bytes0[2]; + + l_s_bytes0[0] = tmps[gid].l_s_bytes[0]; + l_s_bytes0[1] = tmps[gid].l_s_bytes[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + u32 wpc_len[8]; + + wpc_len[0] = 64 + 0 + 0 + pw_len; + wpc_len[1] = pw_len + 0 + 0 + 64; + wpc_len[2] = 64 + salt_len + 0 + pw_len; + wpc_len[3] = pw_len + salt_len + 0 + 64; + wpc_len[4] = 64 + 0 + pw_len + pw_len; + wpc_len[5] = pw_len + 0 + pw_len + 64; + wpc_len[6] = 64 + salt_len + pw_len + pw_len; + wpc_len[7] = pw_len + salt_len + pw_len + 64; + + u64 wpc[8][16] = { { 0 } }; + + for (u32 i = 0; i < 8; i++) + { + u32 block_len = 0; + + if (i & 1) + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); + } + } + else + { + block_len += 64; + } + + if (i & 2) + { + for (u32 j = 0; j < salt_len; j++) + { + PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_s_bytes0, j)); + } + } + + if (i & 4) + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); + } + } + + if (i & 1) + { + block_len += 64; + } + else + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); + } + } + + PUTCHAR64_BE (wpc[i], block_len, 0x80); + + wpc[i][15] = block_len * 8; + } + + /** + * base + */ + + u64 l_alt_result[8]; + + l_alt_result[0] = tmps[gid].l_alt_result[0]; + l_alt_result[1] = tmps[gid].l_alt_result[1]; + l_alt_result[2] = tmps[gid].l_alt_result[2]; + l_alt_result[3] = tmps[gid].l_alt_result[3]; + l_alt_result[4] = tmps[gid].l_alt_result[4]; + l_alt_result[5] = tmps[gid].l_alt_result[5]; + l_alt_result[6] = tmps[gid].l_alt_result[6]; + l_alt_result[7] = tmps[gid].l_alt_result[7]; + + /* Repeatedly run the collected hash value through SHA512 to burn + CPU cycles. */ + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 2 : 0; + const u32 j7 = (j % 7) ? 4 : 0; + + const u32 pc = j1 + j3 + j7; + + u64 block[16]; + + block[ 0] = wpc[pc][ 0]; + block[ 1] = wpc[pc][ 1]; + block[ 2] = wpc[pc][ 2]; + block[ 3] = wpc[pc][ 3]; + block[ 4] = wpc[pc][ 4]; + block[ 5] = wpc[pc][ 5]; + block[ 6] = wpc[pc][ 6]; + block[ 7] = wpc[pc][ 7]; + block[ 8] = wpc[pc][ 8]; + block[ 9] = wpc[pc][ 9]; + block[10] = wpc[pc][10]; + block[11] = wpc[pc][11]; + block[12] = wpc[pc][12]; + block[13] = wpc[pc][13]; + block[14] = wpc[pc][14]; + block[15] = wpc[pc][15]; + + if (j1) + { + const u32 block_len = wpc_len[pc]; + + #ifdef _unroll + #pragma unroll + #endif + for (u32 k = 0, p = block_len - 64; k < 64; k++, p++) + { + PUTCHAR64_BE (block, p, GETCHAR64_BE (l_alt_result, k)); + } + } + else + { + block[0] = l_alt_result[0]; + block[1] = l_alt_result[1]; + block[2] = l_alt_result[2]; + block[3] = l_alt_result[3]; + block[4] = l_alt_result[4]; + block[5] = l_alt_result[5]; + block[6] = l_alt_result[6]; + block[7] = l_alt_result[7]; + } + + l_alt_result[0] = SHA512M_A; + l_alt_result[1] = SHA512M_B; + l_alt_result[2] = SHA512M_C; + l_alt_result[3] = SHA512M_D; + l_alt_result[4] = SHA512M_E; + l_alt_result[5] = SHA512M_F; + l_alt_result[6] = SHA512M_G; + l_alt_result[7] = SHA512M_H; + + sha512_transform (block, l_alt_result); + } + + tmps[gid].l_alt_result[0] = l_alt_result[0]; + tmps[gid].l_alt_result[1] = l_alt_result[1]; + tmps[gid].l_alt_result[2] = l_alt_result[2]; + tmps[gid].l_alt_result[3] = l_alt_result[3]; + tmps[gid].l_alt_result[4] = l_alt_result[4]; + tmps[gid].l_alt_result[5] = l_alt_result[5]; + tmps[gid].l_alt_result[6] = l_alt_result[6]; + tmps[gid].l_alt_result[7] = l_alt_result[7]; +} + +__kernel void m01800_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + const u64 a = swap64 (tmps[gid].l_alt_result[0]); + const u64 b = swap64 (tmps[gid].l_alt_result[1]); + + const u32 r0 = l32_from_64 (a); + const u32 r1 = h32_from_64 (a); + const u32 r2 = l32_from_64 (b); + const u32 r3 = h32_from_64 (b); + + #define il_pos 0 + + #include COMPARE_M +} diff --git a/OpenCL/m01800-pure.cl b/OpenCL/m01800-pure.cl deleted file mode 100644 index 46270e3dc..000000000 --- a/OpenCL/m01800-pure.cl +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_sha512.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -__kernel void m01800_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32 (w[idx]); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32 (s[idx]); - } - - /** - * prepare - */ - - sha512_ctx_t ctx; - - sha512_init (&ctx); - - sha512_update (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - sha512_update (&ctx, w, pw_len); - - sha512_final (&ctx); - - u32 final[32] = { 0 }; - - final[ 0] = h32_from_64_S (ctx.h[0]); - final[ 1] = l32_from_64_S (ctx.h[0]); - final[ 2] = h32_from_64_S (ctx.h[1]); - final[ 3] = l32_from_64_S (ctx.h[1]); - final[ 4] = h32_from_64_S (ctx.h[2]); - final[ 5] = l32_from_64_S (ctx.h[2]); - final[ 6] = h32_from_64_S (ctx.h[3]); - final[ 7] = l32_from_64_S (ctx.h[3]); - final[ 8] = h32_from_64_S (ctx.h[4]); - final[ 9] = l32_from_64_S (ctx.h[4]); - final[10] = h32_from_64_S (ctx.h[5]); - final[11] = l32_from_64_S (ctx.h[5]); - final[12] = h32_from_64_S (ctx.h[6]); - final[13] = l32_from_64_S (ctx.h[6]); - final[14] = h32_from_64_S (ctx.h[7]); - final[15] = l32_from_64_S (ctx.h[7]); - - // alt_result - - sha512_init (&ctx); - - sha512_update (&ctx, w, pw_len); - - sha512_update (&ctx, s, salt_len); - - int pl; - - for (pl = pw_len; pl > 64; pl -= 64) - { - sha512_update (&ctx, final, 64); - } - - u32 t_final[32] = { 0 }; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 16; i++) t_final[i] = final[i]; - - truncate_block_16x4_be (t_final + 0, t_final + 4, t_final + 8, t_final + 12, pl); - - sha512_update (&ctx, t_final, pl); - - for (int cnt = pw_len; cnt > 0; cnt >>= 1) - { - if ((cnt & 1) != 0) - { - sha512_update (&ctx, final, 64); - } - else - { - sha512_update (&ctx, w, pw_len); - } - } - - sha512_final (&ctx); - - tmps[gid].alt_result[ 0] = h32_from_64_S (ctx.h[0]); - tmps[gid].alt_result[ 1] = l32_from_64_S (ctx.h[0]); - tmps[gid].alt_result[ 2] = h32_from_64_S (ctx.h[1]); - tmps[gid].alt_result[ 3] = l32_from_64_S (ctx.h[1]); - tmps[gid].alt_result[ 4] = h32_from_64_S (ctx.h[2]); - tmps[gid].alt_result[ 5] = l32_from_64_S (ctx.h[2]); - tmps[gid].alt_result[ 6] = h32_from_64_S (ctx.h[3]); - tmps[gid].alt_result[ 7] = l32_from_64_S (ctx.h[3]); - tmps[gid].alt_result[ 8] = h32_from_64_S (ctx.h[4]); - tmps[gid].alt_result[ 9] = l32_from_64_S (ctx.h[4]); - tmps[gid].alt_result[10] = h32_from_64_S (ctx.h[5]); - tmps[gid].alt_result[11] = l32_from_64_S (ctx.h[5]); - tmps[gid].alt_result[12] = h32_from_64_S (ctx.h[6]); - tmps[gid].alt_result[13] = l32_from_64_S (ctx.h[6]); - tmps[gid].alt_result[14] = h32_from_64_S (ctx.h[7]); - tmps[gid].alt_result[15] = l32_from_64_S (ctx.h[7]); - - // p_bytes - - sha512_init (&ctx); - - for (u32 j = 0; j < pw_len; j++) - { - sha512_update (&ctx, w, pw_len); - } - - sha512_final (&ctx); - - final[ 0] = h32_from_64_S (ctx.h[0]); - final[ 1] = l32_from_64_S (ctx.h[0]); - final[ 2] = h32_from_64_S (ctx.h[1]); - final[ 3] = l32_from_64_S (ctx.h[1]); - final[ 4] = h32_from_64_S (ctx.h[2]); - final[ 5] = l32_from_64_S (ctx.h[2]); - final[ 6] = h32_from_64_S (ctx.h[3]); - final[ 7] = l32_from_64_S (ctx.h[3]); - final[ 8] = h32_from_64_S (ctx.h[4]); - final[ 9] = l32_from_64_S (ctx.h[4]); - final[10] = h32_from_64_S (ctx.h[5]); - final[11] = l32_from_64_S (ctx.h[5]); - final[12] = h32_from_64_S (ctx.h[6]); - final[13] = l32_from_64_S (ctx.h[6]); - final[14] = h32_from_64_S (ctx.h[7]); - final[15] = l32_from_64_S (ctx.h[7]); - - u32 p_final[64] = { 0 }; - - int idx; - - for (pl = pw_len, idx = 0; pl > 64; pl -= 64, idx += 16) - { - p_final[idx + 0] = final[ 0]; - p_final[idx + 1] = final[ 1]; - p_final[idx + 2] = final[ 2]; - p_final[idx + 3] = final[ 3]; - p_final[idx + 4] = final[ 4]; - p_final[idx + 5] = final[ 5]; - p_final[idx + 6] = final[ 6]; - p_final[idx + 7] = final[ 7]; - p_final[idx + 8] = final[ 8]; - p_final[idx + 9] = final[ 9]; - p_final[idx + 10] = final[10]; - p_final[idx + 11] = final[11]; - p_final[idx + 12] = final[12]; - p_final[idx + 13] = final[13]; - p_final[idx + 14] = final[14]; - p_final[idx + 15] = final[15]; - } - - truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); - - p_final[idx + 0] = final[ 0]; - p_final[idx + 1] = final[ 1]; - p_final[idx + 2] = final[ 2]; - p_final[idx + 3] = final[ 3]; - p_final[idx + 4] = final[ 4]; - p_final[idx + 5] = final[ 5]; - p_final[idx + 6] = final[ 6]; - p_final[idx + 7] = final[ 7]; - p_final[idx + 8] = final[ 8]; - p_final[idx + 9] = final[ 9]; - p_final[idx + 10] = final[10]; - p_final[idx + 11] = final[11]; - p_final[idx + 12] = final[12]; - p_final[idx + 13] = final[13]; - p_final[idx + 14] = final[14]; - p_final[idx + 15] = final[15]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) tmps[gid].p_bytes[i] = p_final[i]; - - // s_bytes - - sha512_init (&ctx); - - for (u32 j = 0; j < 16 + (tmps[gid].alt_result[0] >> 24); j++) - { - sha512_update (&ctx, s, salt_len); - } - - sha512_final (&ctx); - - final[ 0] = h32_from_64_S (ctx.h[0]); - final[ 1] = l32_from_64_S (ctx.h[0]); - final[ 2] = h32_from_64_S (ctx.h[1]); - final[ 3] = l32_from_64_S (ctx.h[1]); - final[ 4] = h32_from_64_S (ctx.h[2]); - final[ 5] = l32_from_64_S (ctx.h[2]); - final[ 6] = h32_from_64_S (ctx.h[3]); - final[ 7] = l32_from_64_S (ctx.h[3]); - final[ 8] = h32_from_64_S (ctx.h[4]); - final[ 9] = l32_from_64_S (ctx.h[4]); - final[10] = h32_from_64_S (ctx.h[5]); - final[11] = l32_from_64_S (ctx.h[5]); - final[12] = h32_from_64_S (ctx.h[6]); - final[13] = l32_from_64_S (ctx.h[6]); - final[14] = h32_from_64_S (ctx.h[7]); - final[15] = l32_from_64_S (ctx.h[7]); - - u32 s_final[64] = { 0 }; - - for (pl = salt_len, idx = 0; pl > 64; pl -= 64, idx += 16) - { - s_final[idx + 0] = final[ 0]; - s_final[idx + 1] = final[ 1]; - s_final[idx + 2] = final[ 2]; - s_final[idx + 3] = final[ 3]; - s_final[idx + 4] = final[ 4]; - s_final[idx + 5] = final[ 5]; - s_final[idx + 6] = final[ 6]; - s_final[idx + 7] = final[ 7]; - s_final[idx + 8] = final[ 8]; - s_final[idx + 9] = final[ 9]; - s_final[idx + 10] = final[10]; - s_final[idx + 11] = final[11]; - s_final[idx + 12] = final[12]; - s_final[idx + 13] = final[13]; - s_final[idx + 14] = final[14]; - s_final[idx + 15] = final[15]; - } - - truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); - - s_final[idx + 0] = final[ 0]; - s_final[idx + 1] = final[ 1]; - s_final[idx + 2] = final[ 2]; - s_final[idx + 3] = final[ 3]; - s_final[idx + 4] = final[ 4]; - s_final[idx + 5] = final[ 5]; - s_final[idx + 6] = final[ 6]; - s_final[idx + 7] = final[ 7]; - s_final[idx + 8] = final[ 8]; - s_final[idx + 9] = final[ 9]; - s_final[idx + 10] = final[10]; - s_final[idx + 11] = final[11]; - s_final[idx + 12] = final[12]; - s_final[idx + 13] = final[13]; - s_final[idx + 14] = final[14]; - s_final[idx + 15] = final[15]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) tmps[gid].s_bytes[i] = s_final[i]; -} - -__kernel void m01800_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 pw_len = pws[gid].pw_len; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - u32 alt_result[32] = { 0 }; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 16; i++) alt_result[i] = tmps[gid].alt_result[i]; - - /* Repeatedly run the collected hash value through SHA512 to burn - CPU cycles. */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - sha512_ctx_t ctx; - - sha512_init (&ctx); - - if (j & 1) - { - sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - else - { - sha512_update (&ctx, alt_result, 64); - } - - if (j % 3) - { - sha512_update_global (&ctx, tmps[gid].s_bytes, salt_len); - } - - if (j % 7) - { - sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - - if (j & 1) - { - sha512_update (&ctx, alt_result, 64); - } - else - { - sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - - sha512_final (&ctx); - - alt_result[ 0] = h32_from_64_S (ctx.h[0]); - alt_result[ 1] = l32_from_64_S (ctx.h[0]); - alt_result[ 2] = h32_from_64_S (ctx.h[1]); - alt_result[ 3] = l32_from_64_S (ctx.h[1]); - alt_result[ 4] = h32_from_64_S (ctx.h[2]); - alt_result[ 5] = l32_from_64_S (ctx.h[2]); - alt_result[ 6] = h32_from_64_S (ctx.h[3]); - alt_result[ 7] = l32_from_64_S (ctx.h[3]); - alt_result[ 8] = h32_from_64_S (ctx.h[4]); - alt_result[ 9] = l32_from_64_S (ctx.h[4]); - alt_result[10] = h32_from_64_S (ctx.h[5]); - alt_result[11] = l32_from_64_S (ctx.h[5]); - alt_result[12] = h32_from_64_S (ctx.h[6]); - alt_result[13] = l32_from_64_S (ctx.h[6]); - alt_result[14] = h32_from_64_S (ctx.h[7]); - alt_result[15] = l32_from_64_S (ctx.h[7]); - } - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 16; i++) tmps[gid].alt_result[i] = alt_result[i]; -} - -__kernel void m01800_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - const u32 r0 = swap32 (tmps[gid].alt_result[0]); - const u32 r1 = swap32 (tmps[gid].alt_result[1]); - const u32 r2 = swap32 (tmps[gid].alt_result[2]); - const u32 r3 = swap32 (tmps[gid].alt_result[3]); - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m01800.cl b/OpenCL/m01800.cl index 8ea852c61..46270e3dc 100644 --- a/OpenCL/m01800.cl +++ b/OpenCL/m01800.cl @@ -8,227 +8,11 @@ #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_sha512.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" -// Buggy drivers... - -#ifdef IS_AMD -#define STATE_DECL volatile -//#define STATE_DECL -#else -#define STATE_DECL -#endif - -#define PUTCHAR64_BE(a,p,c) ((u8 *)(a))[(p) ^ 7] = (u8) (c) -#define GETCHAR64_BE(a,p) ((u8 *)(a))[(p) ^ 7] - -typedef struct -{ - u64 state[8]; - u64 buf[16]; - int len; - -} sha512_ctx_t; - -__constant u64a k_sha512[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha512_transform (const u64 *w, u64 *digest) -{ - u64 w0_t = w[ 0]; - u64 w1_t = w[ 1]; - u64 w2_t = w[ 2]; - u64 w3_t = w[ 3]; - u64 w4_t = w[ 4]; - u64 w5_t = w[ 5]; - u64 w6_t = w[ 6]; - u64 w7_t = w[ 7]; - u64 w8_t = w[ 8]; - u64 w9_t = w[ 9]; - u64 wa_t = w[10]; - u64 wb_t = w[11]; - u64 wc_t = w[12]; - u64 wd_t = w[13]; - u64 we_t = w[14]; - u64 wf_t = w[15]; - - STATE_DECL u64 a = digest[0]; - STATE_DECL u64 b = digest[1]; - STATE_DECL u64 c = digest[2]; - STATE_DECL u64 d = digest[3]; - STATE_DECL u64 e = digest[4]; - STATE_DECL u64 f = digest[5]; - STATE_DECL u64 g = digest[6]; - STATE_DECL u64 h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void sha512_init (sha512_ctx_t *sha512_ctx) -{ - sha512_ctx->state[0] = SHA512M_A; - sha512_ctx->state[1] = SHA512M_B; - sha512_ctx->state[2] = SHA512M_C; - sha512_ctx->state[3] = SHA512M_D; - sha512_ctx->state[4] = SHA512M_E; - sha512_ctx->state[5] = SHA512M_F; - sha512_ctx->state[6] = SHA512M_G; - sha512_ctx->state[7] = SHA512M_H; - - sha512_ctx->len = 0; -} - -void sha512_update (sha512_ctx_t *sha512_ctx, const u64 *buf, int len) -{ - int pos = sha512_ctx->len & 0x7f; - - sha512_ctx->len += len; - - if ((pos + len) < 128) - { - for (int i = 0; i < len; i++) - { - PUTCHAR64_BE (sha512_ctx->buf, pos++, GETCHAR64_BE (buf, i)); - } - - return; - } - - int cnt = 128 - pos; - - for (int i = 0; i < cnt; i++) - { - PUTCHAR64_BE (sha512_ctx->buf, pos++, GETCHAR64_BE (buf, i)); - } - - sha512_transform (sha512_ctx->buf, sha512_ctx->state); - - len -= cnt; - - for (int i = 0; i < len; i++) - { - PUTCHAR64_BE (sha512_ctx->buf, i, GETCHAR64_BE (buf, cnt + i)); - } -} - -void sha512_final (sha512_ctx_t *sha512_ctx) -{ - int pos = sha512_ctx->len & 0x7f; - - for (int i = pos; i < 128; i++) - { - PUTCHAR64_BE (sha512_ctx->buf, i, 0); - } - - PUTCHAR64_BE (sha512_ctx->buf, pos, 0x80); - - if (pos >= 112) - { - sha512_transform (sha512_ctx->buf, sha512_ctx->state); - - sha512_ctx->buf[ 0] = 0; - sha512_ctx->buf[ 1] = 0; - sha512_ctx->buf[ 2] = 0; - sha512_ctx->buf[ 3] = 0; - sha512_ctx->buf[ 4] = 0; - sha512_ctx->buf[ 5] = 0; - sha512_ctx->buf[ 6] = 0; - sha512_ctx->buf[ 7] = 0; - sha512_ctx->buf[ 8] = 0; - sha512_ctx->buf[ 9] = 0; - sha512_ctx->buf[10] = 0; - sha512_ctx->buf[11] = 0; - sha512_ctx->buf[12] = 0; - sha512_ctx->buf[13] = 0; - sha512_ctx->buf[14] = 0; - sha512_ctx->buf[15] = 0; - } - - sha512_ctx->buf[15] = sha512_ctx->len * 8; - - sha512_transform (sha512_ctx->buf, sha512_ctx->state); -} - __kernel void m01800_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -239,123 +23,287 @@ __kernel void m01800_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; + /** + * init + */ const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf[4]; + u32 w[64] = { 0 }; - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; - - u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * buffers - */ - - u64 pw[2]; - - pw[0] = swap64 (hl32_to_64 (w0[1], w0[0])); - pw[1] = swap64 (hl32_to_64 (w0[3], w0[2])); - - u64 salt[2]; - - salt[0] = swap64 (hl32_to_64 (salt_buf[1], salt_buf[0])); - salt[1] = swap64 (hl32_to_64 (salt_buf[3], salt_buf[2])); - - /** - * begin - */ - - sha512_ctx_t sha512_ctx; - - sha512_init (&sha512_ctx); - - sha512_update (&sha512_ctx, pw, pw_len); - sha512_update (&sha512_ctx, salt, salt_len); - sha512_update (&sha512_ctx, pw, pw_len); - - sha512_final (&sha512_ctx); - - u64 tmp[8]; - - tmp[0] = sha512_ctx.state[0]; - tmp[1] = sha512_ctx.state[1]; - tmp[2] = sha512_ctx.state[2]; - tmp[3] = sha512_ctx.state[3]; - tmp[4] = sha512_ctx.state[4]; - tmp[5] = sha512_ctx.state[5]; - tmp[6] = sha512_ctx.state[6]; - tmp[7] = sha512_ctx.state[7]; - - sha512_init (&sha512_ctx); - - sha512_update (&sha512_ctx, pw, pw_len); - sha512_update (&sha512_ctx, salt, salt_len); - sha512_update (&sha512_ctx, tmp, pw_len); - - for (u32 j = pw_len; j; j >>= 1) + for (int idx = 0; idx < pw_lenv; idx++) { - if (j & 1) + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32 (w[idx]); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32 (s[idx]); + } + + /** + * prepare + */ + + sha512_ctx_t ctx; + + sha512_init (&ctx); + + sha512_update (&ctx, w, pw_len); + + sha512_update (&ctx, s, salt_len); + + sha512_update (&ctx, w, pw_len); + + sha512_final (&ctx); + + u32 final[32] = { 0 }; + + final[ 0] = h32_from_64_S (ctx.h[0]); + final[ 1] = l32_from_64_S (ctx.h[0]); + final[ 2] = h32_from_64_S (ctx.h[1]); + final[ 3] = l32_from_64_S (ctx.h[1]); + final[ 4] = h32_from_64_S (ctx.h[2]); + final[ 5] = l32_from_64_S (ctx.h[2]); + final[ 6] = h32_from_64_S (ctx.h[3]); + final[ 7] = l32_from_64_S (ctx.h[3]); + final[ 8] = h32_from_64_S (ctx.h[4]); + final[ 9] = l32_from_64_S (ctx.h[4]); + final[10] = h32_from_64_S (ctx.h[5]); + final[11] = l32_from_64_S (ctx.h[5]); + final[12] = h32_from_64_S (ctx.h[6]); + final[13] = l32_from_64_S (ctx.h[6]); + final[14] = h32_from_64_S (ctx.h[7]); + final[15] = l32_from_64_S (ctx.h[7]); + + // alt_result + + sha512_init (&ctx); + + sha512_update (&ctx, w, pw_len); + + sha512_update (&ctx, s, salt_len); + + int pl; + + for (pl = pw_len; pl > 64; pl -= 64) + { + sha512_update (&ctx, final, 64); + } + + u32 t_final[32] = { 0 }; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 16; i++) t_final[i] = final[i]; + + truncate_block_16x4_be (t_final + 0, t_final + 4, t_final + 8, t_final + 12, pl); + + sha512_update (&ctx, t_final, pl); + + for (int cnt = pw_len; cnt > 0; cnt >>= 1) + { + if ((cnt & 1) != 0) { - sha512_update (&sha512_ctx, tmp, 64); + sha512_update (&ctx, final, 64); } else { - sha512_update (&sha512_ctx, pw, pw_len); + sha512_update (&ctx, w, pw_len); } } - sha512_final (&sha512_ctx); + sha512_final (&ctx); - tmps[gid].l_alt_result[0] = sha512_ctx.state[0]; - tmps[gid].l_alt_result[1] = sha512_ctx.state[1]; - tmps[gid].l_alt_result[2] = sha512_ctx.state[2]; - tmps[gid].l_alt_result[3] = sha512_ctx.state[3]; - tmps[gid].l_alt_result[4] = sha512_ctx.state[4]; - tmps[gid].l_alt_result[5] = sha512_ctx.state[5]; - tmps[gid].l_alt_result[6] = sha512_ctx.state[6]; - tmps[gid].l_alt_result[7] = sha512_ctx.state[7]; + tmps[gid].alt_result[ 0] = h32_from_64_S (ctx.h[0]); + tmps[gid].alt_result[ 1] = l32_from_64_S (ctx.h[0]); + tmps[gid].alt_result[ 2] = h32_from_64_S (ctx.h[1]); + tmps[gid].alt_result[ 3] = l32_from_64_S (ctx.h[1]); + tmps[gid].alt_result[ 4] = h32_from_64_S (ctx.h[2]); + tmps[gid].alt_result[ 5] = l32_from_64_S (ctx.h[2]); + tmps[gid].alt_result[ 6] = h32_from_64_S (ctx.h[3]); + tmps[gid].alt_result[ 7] = l32_from_64_S (ctx.h[3]); + tmps[gid].alt_result[ 8] = h32_from_64_S (ctx.h[4]); + tmps[gid].alt_result[ 9] = l32_from_64_S (ctx.h[4]); + tmps[gid].alt_result[10] = h32_from_64_S (ctx.h[5]); + tmps[gid].alt_result[11] = l32_from_64_S (ctx.h[5]); + tmps[gid].alt_result[12] = h32_from_64_S (ctx.h[6]); + tmps[gid].alt_result[13] = l32_from_64_S (ctx.h[6]); + tmps[gid].alt_result[14] = h32_from_64_S (ctx.h[7]); + tmps[gid].alt_result[15] = l32_from_64_S (ctx.h[7]); // p_bytes - sha512_init (&sha512_ctx); + sha512_init (&ctx); for (u32 j = 0; j < pw_len; j++) { - sha512_update (&sha512_ctx, pw, pw_len); + sha512_update (&ctx, w, pw_len); } - sha512_final (&sha512_ctx); + sha512_final (&ctx); - tmps[gid].l_p_bytes[0] = sha512_ctx.state[0]; - tmps[gid].l_p_bytes[1] = sha512_ctx.state[1]; + final[ 0] = h32_from_64_S (ctx.h[0]); + final[ 1] = l32_from_64_S (ctx.h[0]); + final[ 2] = h32_from_64_S (ctx.h[1]); + final[ 3] = l32_from_64_S (ctx.h[1]); + final[ 4] = h32_from_64_S (ctx.h[2]); + final[ 5] = l32_from_64_S (ctx.h[2]); + final[ 6] = h32_from_64_S (ctx.h[3]); + final[ 7] = l32_from_64_S (ctx.h[3]); + final[ 8] = h32_from_64_S (ctx.h[4]); + final[ 9] = l32_from_64_S (ctx.h[4]); + final[10] = h32_from_64_S (ctx.h[5]); + final[11] = l32_from_64_S (ctx.h[5]); + final[12] = h32_from_64_S (ctx.h[6]); + final[13] = l32_from_64_S (ctx.h[6]); + final[14] = h32_from_64_S (ctx.h[7]); + final[15] = l32_from_64_S (ctx.h[7]); + + u32 p_final[64] = { 0 }; + + int idx; + + for (pl = pw_len, idx = 0; pl > 64; pl -= 64, idx += 16) + { + p_final[idx + 0] = final[ 0]; + p_final[idx + 1] = final[ 1]; + p_final[idx + 2] = final[ 2]; + p_final[idx + 3] = final[ 3]; + p_final[idx + 4] = final[ 4]; + p_final[idx + 5] = final[ 5]; + p_final[idx + 6] = final[ 6]; + p_final[idx + 7] = final[ 7]; + p_final[idx + 8] = final[ 8]; + p_final[idx + 9] = final[ 9]; + p_final[idx + 10] = final[10]; + p_final[idx + 11] = final[11]; + p_final[idx + 12] = final[12]; + p_final[idx + 13] = final[13]; + p_final[idx + 14] = final[14]; + p_final[idx + 15] = final[15]; + } + + truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); + + p_final[idx + 0] = final[ 0]; + p_final[idx + 1] = final[ 1]; + p_final[idx + 2] = final[ 2]; + p_final[idx + 3] = final[ 3]; + p_final[idx + 4] = final[ 4]; + p_final[idx + 5] = final[ 5]; + p_final[idx + 6] = final[ 6]; + p_final[idx + 7] = final[ 7]; + p_final[idx + 8] = final[ 8]; + p_final[idx + 9] = final[ 9]; + p_final[idx + 10] = final[10]; + p_final[idx + 11] = final[11]; + p_final[idx + 12] = final[12]; + p_final[idx + 13] = final[13]; + p_final[idx + 14] = final[14]; + p_final[idx + 15] = final[15]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) tmps[gid].p_bytes[i] = p_final[i]; // s_bytes - sha512_init (&sha512_ctx); + sha512_init (&ctx); - for (u32 j = 0; j < 16 + ((tmps[gid].l_alt_result[0] >> 56) & 0xff); j++) + for (u32 j = 0; j < 16 + (tmps[gid].alt_result[0] >> 24); j++) { - sha512_update (&sha512_ctx, salt, salt_len); + sha512_update (&ctx, s, salt_len); } - sha512_final (&sha512_ctx); + sha512_final (&ctx); - tmps[gid].l_s_bytes[0] = sha512_ctx.state[0]; - tmps[gid].l_s_bytes[1] = sha512_ctx.state[1]; + final[ 0] = h32_from_64_S (ctx.h[0]); + final[ 1] = l32_from_64_S (ctx.h[0]); + final[ 2] = h32_from_64_S (ctx.h[1]); + final[ 3] = l32_from_64_S (ctx.h[1]); + final[ 4] = h32_from_64_S (ctx.h[2]); + final[ 5] = l32_from_64_S (ctx.h[2]); + final[ 6] = h32_from_64_S (ctx.h[3]); + final[ 7] = l32_from_64_S (ctx.h[3]); + final[ 8] = h32_from_64_S (ctx.h[4]); + final[ 9] = l32_from_64_S (ctx.h[4]); + final[10] = h32_from_64_S (ctx.h[5]); + final[11] = l32_from_64_S (ctx.h[5]); + final[12] = h32_from_64_S (ctx.h[6]); + final[13] = l32_from_64_S (ctx.h[6]); + final[14] = h32_from_64_S (ctx.h[7]); + final[15] = l32_from_64_S (ctx.h[7]); + + u32 s_final[64] = { 0 }; + + for (pl = salt_len, idx = 0; pl > 64; pl -= 64, idx += 16) + { + s_final[idx + 0] = final[ 0]; + s_final[idx + 1] = final[ 1]; + s_final[idx + 2] = final[ 2]; + s_final[idx + 3] = final[ 3]; + s_final[idx + 4] = final[ 4]; + s_final[idx + 5] = final[ 5]; + s_final[idx + 6] = final[ 6]; + s_final[idx + 7] = final[ 7]; + s_final[idx + 8] = final[ 8]; + s_final[idx + 9] = final[ 9]; + s_final[idx + 10] = final[10]; + s_final[idx + 11] = final[11]; + s_final[idx + 12] = final[12]; + s_final[idx + 13] = final[13]; + s_final[idx + 14] = final[14]; + s_final[idx + 15] = final[15]; + } + + truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); + + s_final[idx + 0] = final[ 0]; + s_final[idx + 1] = final[ 1]; + s_final[idx + 2] = final[ 2]; + s_final[idx + 3] = final[ 3]; + s_final[idx + 4] = final[ 4]; + s_final[idx + 5] = final[ 5]; + s_final[idx + 6] = final[ 6]; + s_final[idx + 7] = final[ 7]; + s_final[idx + 8] = final[ 8]; + s_final[idx + 9] = final[ 9]; + s_final[idx + 10] = final[10]; + s_final[idx + 11] = final[11]; + s_final[idx + 12] = final[12]; + s_final[idx + 13] = final[13]; + s_final[idx + 14] = final[14]; + s_final[idx + 15] = final[15]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) tmps[gid].s_bytes[i] = s_final[i]; } __kernel void m01800_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -368,171 +316,78 @@ __kernel void m01800_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u64 l_p_bytes0[2]; - - l_p_bytes0[0] = tmps[gid].l_p_bytes[0]; - l_p_bytes0[1] = tmps[gid].l_p_bytes[1]; - const u32 pw_len = pws[gid].pw_len; - u64 l_s_bytes0[2]; - - l_s_bytes0[0] = tmps[gid].l_s_bytes[0]; - l_s_bytes0[1] = tmps[gid].l_s_bytes[1]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 wpc_len[8]; + u32 alt_result[32] = { 0 }; - wpc_len[0] = 64 + 0 + 0 + pw_len; - wpc_len[1] = pw_len + 0 + 0 + 64; - wpc_len[2] = 64 + salt_len + 0 + pw_len; - wpc_len[3] = pw_len + salt_len + 0 + 64; - wpc_len[4] = 64 + 0 + pw_len + pw_len; - wpc_len[5] = pw_len + 0 + pw_len + 64; - wpc_len[6] = 64 + salt_len + pw_len + pw_len; - wpc_len[7] = pw_len + salt_len + pw_len + 64; - - u64 wpc[8][16] = { { 0 } }; - - for (u32 i = 0; i < 8; i++) - { - u32 block_len = 0; - - if (i & 1) - { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); - } - } - else - { - block_len += 64; - } - - if (i & 2) - { - for (u32 j = 0; j < salt_len; j++) - { - PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_s_bytes0, j)); - } - } - - if (i & 4) - { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); - } - } - - if (i & 1) - { - block_len += 64; - } - else - { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR64_BE (wpc[i], block_len++, GETCHAR64_BE (l_p_bytes0, j)); - } - } - - PUTCHAR64_BE (wpc[i], block_len, 0x80); - - wpc[i][15] = block_len * 8; - } - - /** - * base - */ - - u64 l_alt_result[8]; - - l_alt_result[0] = tmps[gid].l_alt_result[0]; - l_alt_result[1] = tmps[gid].l_alt_result[1]; - l_alt_result[2] = tmps[gid].l_alt_result[2]; - l_alt_result[3] = tmps[gid].l_alt_result[3]; - l_alt_result[4] = tmps[gid].l_alt_result[4]; - l_alt_result[5] = tmps[gid].l_alt_result[5]; - l_alt_result[6] = tmps[gid].l_alt_result[6]; - l_alt_result[7] = tmps[gid].l_alt_result[7]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 16; i++) alt_result[i] = tmps[gid].alt_result[i]; /* Repeatedly run the collected hash value through SHA512 to burn CPU cycles. */ for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 2 : 0; - const u32 j7 = (j % 7) ? 4 : 0; + sha512_ctx_t ctx; - const u32 pc = j1 + j3 + j7; + sha512_init (&ctx); - u64 block[16]; - - block[ 0] = wpc[pc][ 0]; - block[ 1] = wpc[pc][ 1]; - block[ 2] = wpc[pc][ 2]; - block[ 3] = wpc[pc][ 3]; - block[ 4] = wpc[pc][ 4]; - block[ 5] = wpc[pc][ 5]; - block[ 6] = wpc[pc][ 6]; - block[ 7] = wpc[pc][ 7]; - block[ 8] = wpc[pc][ 8]; - block[ 9] = wpc[pc][ 9]; - block[10] = wpc[pc][10]; - block[11] = wpc[pc][11]; - block[12] = wpc[pc][12]; - block[13] = wpc[pc][13]; - block[14] = wpc[pc][14]; - block[15] = wpc[pc][15]; - - if (j1) + if (j & 1) { - const u32 block_len = wpc_len[pc]; - - #ifdef _unroll - #pragma unroll - #endif - for (u32 k = 0, p = block_len - 64; k < 64; k++, p++) - { - PUTCHAR64_BE (block, p, GETCHAR64_BE (l_alt_result, k)); - } + sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); } - else + else { - block[0] = l_alt_result[0]; - block[1] = l_alt_result[1]; - block[2] = l_alt_result[2]; - block[3] = l_alt_result[3]; - block[4] = l_alt_result[4]; - block[5] = l_alt_result[5]; - block[6] = l_alt_result[6]; - block[7] = l_alt_result[7]; + sha512_update (&ctx, alt_result, 64); } - l_alt_result[0] = SHA512M_A; - l_alt_result[1] = SHA512M_B; - l_alt_result[2] = SHA512M_C; - l_alt_result[3] = SHA512M_D; - l_alt_result[4] = SHA512M_E; - l_alt_result[5] = SHA512M_F; - l_alt_result[6] = SHA512M_G; - l_alt_result[7] = SHA512M_H; + if (j % 3) + { + sha512_update_global (&ctx, tmps[gid].s_bytes, salt_len); + } - sha512_transform (block, l_alt_result); + if (j % 7) + { + sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); + } + + if (j & 1) + { + sha512_update (&ctx, alt_result, 64); + } + else + { + sha512_update_global (&ctx, tmps[gid].p_bytes, pw_len); + } + + sha512_final (&ctx); + + alt_result[ 0] = h32_from_64_S (ctx.h[0]); + alt_result[ 1] = l32_from_64_S (ctx.h[0]); + alt_result[ 2] = h32_from_64_S (ctx.h[1]); + alt_result[ 3] = l32_from_64_S (ctx.h[1]); + alt_result[ 4] = h32_from_64_S (ctx.h[2]); + alt_result[ 5] = l32_from_64_S (ctx.h[2]); + alt_result[ 6] = h32_from_64_S (ctx.h[3]); + alt_result[ 7] = l32_from_64_S (ctx.h[3]); + alt_result[ 8] = h32_from_64_S (ctx.h[4]); + alt_result[ 9] = l32_from_64_S (ctx.h[4]); + alt_result[10] = h32_from_64_S (ctx.h[5]); + alt_result[11] = l32_from_64_S (ctx.h[5]); + alt_result[12] = h32_from_64_S (ctx.h[6]); + alt_result[13] = l32_from_64_S (ctx.h[6]); + alt_result[14] = h32_from_64_S (ctx.h[7]); + alt_result[15] = l32_from_64_S (ctx.h[7]); } - tmps[gid].l_alt_result[0] = l_alt_result[0]; - tmps[gid].l_alt_result[1] = l_alt_result[1]; - tmps[gid].l_alt_result[2] = l_alt_result[2]; - tmps[gid].l_alt_result[3] = l_alt_result[3]; - tmps[gid].l_alt_result[4] = l_alt_result[4]; - tmps[gid].l_alt_result[5] = l_alt_result[5]; - tmps[gid].l_alt_result[6] = l_alt_result[6]; - tmps[gid].l_alt_result[7] = l_alt_result[7]; + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 16; i++) tmps[gid].alt_result[i] = alt_result[i]; } __kernel void m01800_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha512crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -547,13 +402,10 @@ __kernel void m01800_comp (__global pw_t *pws, __global const kernel_rule_t *rul const u32 lid = get_local_id (0); - const u64 a = swap64 (tmps[gid].l_alt_result[0]); - const u64 b = swap64 (tmps[gid].l_alt_result[1]); - - const u32 r0 = l32_from_64 (a); - const u32 r1 = h32_from_64 (a); - const u32 r2 = l32_from_64 (b); - const u32 r3 = h32_from_64 (b); + const u32 r0 = swap32 (tmps[gid].alt_result[0]); + const u32 r1 = swap32 (tmps[gid].alt_result[1]); + const u32 r2 = swap32 (tmps[gid].alt_result[2]); + const u32 r3 = swap32 (tmps[gid].alt_result[3]); #define il_pos 0 diff --git a/OpenCL/m02610_a0-optimized.cl b/OpenCL/m02610_a0-optimized.cl new file mode 100644 index 000000000..85fd13bd6 --- /dev/null +++ b/OpenCL/m02610_a0-optimized.cl @@ -0,0 +1,606 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m02610_a0-pure.cl b/OpenCL/m02610_a0-pure.cl deleted file mode 100644 index 633ea6f13..000000000 --- a/OpenCL/m02610_a0-pure.cl +++ /dev/null @@ -1,272 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02610_a0.cl b/OpenCL/m02610_a0.cl index 85fd13bd6..633ea6f13 100644 --- a/OpenCL/m02610_a0.cl +++ b/OpenCL/m02610_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,256 +60,90 @@ __kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.len = 32; - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_update (&ctx, s, salt_len); - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; + md5_final (&ctx); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -337,52 +172,6 @@ __kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -395,212 +184,89 @@ __kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.len = 32; - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_update (&ctx, s, salt_len); - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; + md5_final (&ctx); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m02610_a1-optimized.cl b/OpenCL/m02610_a1-optimized.cl new file mode 100644 index 000000000..8449d1796 --- /dev/null +++ b/OpenCL/m02610_a1-optimized.cl @@ -0,0 +1,722 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m02610_a1-pure.cl b/OpenCL/m02610_a1-pure.cl deleted file mode 100644 index 3818eb960..000000000 --- a/OpenCL/m02610_a1-pure.cl +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02610_a1.cl b/OpenCL/m02610_a1.cl index 8449d1796..3818eb960 100644 --- a/OpenCL/m02610_a1.cl +++ b/OpenCL/m02610_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,315 +58,79 @@ __kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + ctx.len = 32; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md5_update (&ctx, s, salt_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_final (&ctx); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -394,52 +159,6 @@ __kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -452,271 +171,78 @@ __kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + ctx.len = 32; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md5_update (&ctx, s, salt_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_final (&ctx); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m02610_a3-optimized.cl b/OpenCL/m02610_a3-optimized.cl new file mode 100644 index 000000000..3546805f8 --- /dev/null +++ b/OpenCL/m02610_a3-optimized.cl @@ -0,0 +1,973 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m02610m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = (32 + salt_len) * 8; + w3_t[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m02610s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = (32 + salt_len) * 8; + w3_t[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + + } +} + +__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m02610_a3-pure.cl b/OpenCL/m02610_a3-pure.cl deleted file mode 100644 index 442dfe070..000000000 --- a/OpenCL/m02610_a3-pure.cl +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02610_a3.cl b/OpenCL/m02610_a3.cl index 3546805f8..442dfe070 100644 --- a/OpenCL/m02610_a3.cl +++ b/OpenCL/m02610_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m02610m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m02610_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,255 +33,121 @@ void m02610m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.len = 32; - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = (32 + salt_len) * 8; - w3_t[3] = 0; + md5_update_vector (&ctx, s, salt_len); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_final_vector (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m02610s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m02610_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -288,34 +155,26 @@ void m02610s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -329,645 +188,95 @@ void m02610s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.len = 32; - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = (32 + salt_len) * 8; - w3_t[3] = 0; + md5_update_vector (&ctx, s, salt_len); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_final_vector (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m02610_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02610_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02610_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02610_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02610_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02610_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02610s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m02810_a0-optimized.cl b/OpenCL/m02810_a0-optimized.cl new file mode 100644 index 000000000..5c944f9a9 --- /dev/null +++ b/OpenCL/m02810_a0-optimized.cl @@ -0,0 +1,778 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + + w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + + w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m02810_a0-pure.cl b/OpenCL/m02810_a0-pure.cl deleted file mode 100644 index 02dfa50fd..000000000 --- a/OpenCL/m02810_a0-pure.cl +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02810_a0.cl b/OpenCL/m02810_a0.cl index 5c944f9a9..02dfa50fd 100644 --- a/OpenCL/m02810_a0.cl +++ b/OpenCL/m02810_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,342 +60,113 @@ __kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -423,52 +195,6 @@ __kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -481,298 +207,112 @@ __kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m02810_a1-optimized.cl b/OpenCL/m02810_a1-optimized.cl new file mode 100644 index 000000000..2b4f4822f --- /dev/null +++ b/OpenCL/m02810_a1-optimized.cl @@ -0,0 +1,894 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + + w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = salt_buf0[0]; + w0[1] = salt_buf0[1]; + w0[2] = salt_buf0[2]; + w0[3] = salt_buf0[3]; + w1[0] = salt_buf1[0]; + w1[1] = salt_buf1[1]; + w1[2] = salt_buf1[2]; + w1[3] = salt_buf1[3]; + + w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m02810_a1-pure.cl b/OpenCL/m02810_a1-pure.cl deleted file mode 100644 index 28b4e271d..000000000 --- a/OpenCL/m02810_a1-pure.cl +++ /dev/null @@ -1,294 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02810_a1.cl b/OpenCL/m02810_a1.cl index 2b4f4822f..28b4e271d 100644 --- a/OpenCL/m02810_a1.cl +++ b/OpenCL/m02810_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,401 +58,102 @@ __kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; + const u32 salt_len = 32; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - const u32 pw_l_len = pws[gid].pw_len; + u32x s[8] = { 0 }; - /** - * salt - */ + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + md5_ctx_t ctx0; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - - w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -480,52 +182,6 @@ __kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -538,357 +194,101 @@ __kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = salt_buf0[0]; - w0[1] = salt_buf0[1]; - w0[2] = salt_buf0[2]; - w0[3] = salt_buf0[3]; - w1[0] = salt_buf1[0]; - w1[1] = salt_buf1[1]; - w1[2] = salt_buf1[2]; - w1[3] = salt_buf1[3]; - - w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m02810_a3-optimized.cl b/OpenCL/m02810_a3-optimized.cl new file mode 100644 index 000000000..55b2e085d --- /dev/null +++ b/OpenCL/m02810_a3-optimized.cl @@ -0,0 +1,1144 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m02810m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + + w2_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m02810s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = salt_buf0[0]; + w0_t[1] = salt_buf0[1]; + w0_t[2] = salt_buf0[2]; + w0_t[3] = salt_buf0[3]; + w1_t[0] = salt_buf1[0]; + w1_t[1] = salt_buf1[1]; + w1_t[2] = salt_buf1[2]; + w1_t[3] = salt_buf1[3]; + + w2_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w2_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w2_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w2_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w3_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w3_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w3_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w3_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m02810_a3-pure.cl b/OpenCL/m02810_a3-pure.cl deleted file mode 100644 index 0a8eff53b..000000000 --- a/OpenCL/m02810_a3-pure.cl +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = s[0]; - ctx.w0[1] = s[1]; - ctx.w0[2] = s[2]; - ctx.w0[3] = s[3]; - ctx.w1[0] = s[4]; - ctx.w1[1] = s[5]; - ctx.w1[2] = s[6]; - ctx.w1[3] = s[7]; - ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m02810_a3.cl b/OpenCL/m02810_a3.cl index 55b2e085d..0a8eff53b 100644 --- a/OpenCL/m02810_a3.cl +++ b/OpenCL/m02810_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m02810m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m02810_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,341 +33,144 @@ void m02810m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w2_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m02810s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m02810_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -374,34 +178,26 @@ void m02810s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -415,730 +211,118 @@ void m02810s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = s[0]; + ctx.w0[1] = s[1]; + ctx.w0[2] = s[2]; + ctx.w0[3] = s[3]; + ctx.w1[0] = s[4]; + ctx.w1[1] = s[5]; + ctx.w1[2] = s[6]; + ctx.w1[3] = s[7]; + ctx.w2[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w2[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w2[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w2[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w3[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w3[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w3[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w3[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w0_t[0] = salt_buf0[0]; - w0_t[1] = salt_buf0[1]; - w0_t[2] = salt_buf0[2]; - w0_t[3] = salt_buf0[3]; - w1_t[0] = salt_buf1[0]; - w1_t[1] = salt_buf1[1]; - w1_t[2] = salt_buf1[2]; - w1_t[3] = salt_buf1[3]; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w2_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w2_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w2_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w2_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w3_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w3_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w3_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w3_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m02810_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02810_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02810_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02810_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02810_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m02810_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m02810s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m03710_a0-optimized.cl b/OpenCL/m03710_a0-optimized.cl new file mode 100644 index 000000000..50ac6a56b --- /dev/null +++ b/OpenCL/m03710_a0-optimized.cl @@ -0,0 +1,668 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03710_a0-pure.cl b/OpenCL/m03710_a0-pure.cl deleted file mode 100644 index 054465f2d..000000000 --- a/OpenCL/m03710_a0-pure.cl +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, s, salt_len); - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, s, salt_len); - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03710_a0.cl b/OpenCL/m03710_a0.cl index 50ac6a56b..054465f2d 100644 --- a/OpenCL/m03710_a0.cl +++ b/OpenCL/m03710_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,145 +60,63 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = 32 + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + md5_update (&ctx, s, salt_len); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -215,7 +134,7 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -224,122 +143,20 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -368,54 +185,6 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = 32 + salt_len; - /** * digest */ @@ -428,101 +197,67 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + md5_update (&ctx, s, salt_len); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -540,7 +275,7 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -549,120 +284,15 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03710_a1-optimized.cl b/OpenCL/m03710_a1-optimized.cl new file mode 100644 index 000000000..5f2dfeae2 --- /dev/null +++ b/OpenCL/m03710_a1-optimized.cl @@ -0,0 +1,784 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03710_a1-pure.cl b/OpenCL/m03710_a1-pure.cl deleted file mode 100644 index b74fa951f..000000000 --- a/OpenCL/m03710_a1-pure.cl +++ /dev/null @@ -1,274 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, s, salt_len); - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - md5_update (&ctx, s, salt_len); - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03710_a1.cl b/OpenCL/m03710_a1.cl index 5f2dfeae2..b74fa951f 100644 --- a/OpenCL/m03710_a1.cl +++ b/OpenCL/m03710_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,204 +58,52 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = 32 + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + md5_update (&ctx, s, salt_len); - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -272,7 +121,7 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -281,122 +130,20 @@ __kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -425,54 +172,6 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = 32 + salt_len; - /** * digest */ @@ -485,160 +184,56 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + md5_update (&ctx, s, salt_len); - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -656,7 +251,7 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -665,120 +260,15 @@ __kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03710_a3-optimized.cl b/OpenCL/m03710_a3-optimized.cl new file mode 100644 index 000000000..8c9a19124 --- /dev/null +++ b/OpenCL/m03710_a3-optimized.cl @@ -0,0 +1,1034 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m03710m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = pw_salt_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m03710s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = pw_salt_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m03710_a3-pure.cl b/OpenCL/m03710_a3-pure.cl deleted file mode 100644 index 8c35c9263..000000000 --- a/OpenCL/m03710_a3-pure.cl +++ /dev/null @@ -1,308 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, s, salt_len); - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - md5_update_vector (&ctx, s, salt_len); - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03710_a3.cl b/OpenCL/m03710_a3.cl index 8c9a19124..8c35c9263 100644 --- a/OpenCL/m03710_a3.cl +++ b/OpenCL/m03710_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m03710m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m03710_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,286 +33,134 @@ void m03710m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = 32 + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_update_vector (&ctx, s, salt_len); - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - /** - * prepend salt - */ + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - w3_t[2] = pw_salt_len * 8; - w3_t[3] = 0; + md5_final_vector (&ctx); - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m03710s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m03710_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -319,36 +168,26 @@ void m03710s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } - const u32 pw_salt_len = 32 + salt_len; + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -362,673 +201,108 @@ void m03710s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_update_vector (&ctx, s, salt_len); - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - /** - * prepend salt - */ + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - w3_t[2] = pw_salt_len * 8; - w3_t[3] = 0; + md5_final_vector (&ctx); - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m03710_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03710_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03710_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03710m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03710_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03710_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03710_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03710s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m03800_a0-optimized.cl b/OpenCL/m03800_a0-optimized.cl new file mode 100644 index 000000000..ffb19aab8 --- /dev/null +++ b/OpenCL/m03800_a0-optimized.cl @@ -0,0 +1,504 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_salt_len); + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + const u32x salt_out_salt_len = salt_len + out_len + salt_len; + + append_0x80_4x4_VV (w0, w1, w2, w3, salt_out_salt_len); + + w3[2] = salt_out_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = out_len + salt_len; + + switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_salt_len); + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + const u32x salt_out_salt_len = salt_len + out_len + salt_len; + + append_0x80_4x4_VV (w0, w1, w2, w3, salt_out_salt_len); + + w3[2] = salt_out_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03800_a0-pure.cl b/OpenCL/m03800_a0-pure.cl deleted file mode 100644 index 42344114b..000000000 --- a/OpenCL/m03800_a0-pure.cl +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx = ctx0; - - md5_update (&ctx, w, pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03800_a0.cl b/OpenCL/m03800_a0.cl index ffb19aab8..42344114b 100644 --- a/OpenCL/m03800_a0.cl +++ b/OpenCL/m03800_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,303 +12,92 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - /** - * prepend salt - */ + md5_update (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + md5_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_salt_len); - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - const u32x salt_out_salt_len = salt_len + out_len + salt_len; - - append_0x80_4x4_VV (w0, w1, w2, w3, salt_out_salt_len); - - w3[2] = salt_out_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -321,184 +110,63 @@ __kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx = ctx0; - /** - * prepend salt - */ + md5_update (&ctx, w, pw_len); - const u32x out_salt_len = out_len + salt_len; + md5_update (&ctx, s, salt_len); - switch_buffer_by_offset_le_VV (w0, w1, w2, w3, salt_len); + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_salt_len); - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - const u32x salt_out_salt_len = salt_len + out_len + salt_len; - - append_0x80_4x4_VV (w0, w1, w2, w3, salt_out_salt_len); - - w3[2] = salt_out_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03800_a1-optimized.cl b/OpenCL/m03800_a1-optimized.cl new file mode 100644 index 000000000..7c2c0742f --- /dev/null +++ b/OpenCL/m03800_a1-optimized.cl @@ -0,0 +1,622 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_salt_len); + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + const u32x salt_pw_salt_len = salt_len + pw_len + salt_len; + + append_0x80_4x4_VV (w0, w1, w2, w3, salt_pw_salt_len); + + w3[2] = salt_pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0[0]; + s0[1] = salt_buf0[1]; + s0[2] = salt_buf0[2]; + s0[3] = salt_buf0[3]; + s1[0] = salt_buf1[0]; + s1[1] = salt_buf1[1]; + s1[2] = salt_buf1[2]; + s1[3] = salt_buf1[3]; + s2[0] = salt_buf2[0]; + s2[1] = salt_buf2[1]; + s2[2] = salt_buf2[2]; + s2[3] = salt_buf2[3]; + s3[0] = salt_buf3[0]; + s3[1] = salt_buf3[1]; + s3[2] = salt_buf3[2]; + s3[3] = salt_buf3[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_salt_len); + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] |= s3[2]; + w3[3] |= s3[3]; + + const u32x salt_pw_salt_len = salt_len + pw_len + salt_len; + + append_0x80_4x4_VV (w0, w1, w2, w3, salt_pw_salt_len); + + w3[2] = salt_pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03800_a1-pure.cl b/OpenCL/m03800_a1-pure.cl deleted file mode 100644 index c594eee3d..000000000 --- a/OpenCL/m03800_a1-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx = ctx0; - - md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03800_a1.cl b/OpenCL/m03800_a1.cl index 7c2c0742f..c594eee3d 100644 --- a/OpenCL/m03800_a1.cl +++ b/OpenCL/m03800_a1.cl @@ -3,370 +3,86 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" -__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_salt_len); - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - const u32x salt_pw_salt_len = salt_len + pw_len + salt_len; - - append_0x80_4x4_VV (w0, w1, w2, w3, salt_pw_salt_len); - - w3[2] = salt_pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -379,244 +95,50 @@ __kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0[0]; - s0[1] = salt_buf0[1]; - s0[2] = salt_buf0[2]; - s0[3] = salt_buf0[3]; - s1[0] = salt_buf1[0]; - s1[1] = salt_buf1[1]; - s1[2] = salt_buf1[2]; - s1[3] = salt_buf1[3]; - s2[0] = salt_buf2[0]; - s2[1] = salt_buf2[1]; - s2[2] = salt_buf2[2]; - s2[3] = salt_buf2[3]; - s3[0] = salt_buf3[0]; - s3[1] = salt_buf3[1]; - s3[2] = salt_buf3[2]; - s3[3] = salt_buf3[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_salt_len); - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] |= s3[2]; - w3[3] |= s3[3]; - - const u32x salt_pw_salt_len = salt_len + pw_len + salt_len; - - append_0x80_4x4_VV (w0, w1, w2, w3, salt_pw_salt_len); - - w3[2] = salt_pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03800_a3-optimized.cl b/OpenCL/m03800_a3-optimized.cl new file mode 100644 index 000000000..fb704eef0 --- /dev/null +++ b/OpenCL/m03800_a3-optimized.cl @@ -0,0 +1,790 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +void m03800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + const u32 salt_pw_salt_len = salt_len + pw_len + salt_len; + + switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_salt_len); + + salt_buf0[0] |= salt_buf0_t[0]; + salt_buf0[1] |= salt_buf0_t[1]; + salt_buf0[2] |= salt_buf0_t[2]; + salt_buf0[3] |= salt_buf0_t[3]; + salt_buf1[0] |= salt_buf1_t[0]; + salt_buf1[1] |= salt_buf1_t[1]; + salt_buf1[2] |= salt_buf1_t[2]; + salt_buf1[3] |= salt_buf1_t[3]; + salt_buf2[0] |= salt_buf2_t[0]; + salt_buf2[1] |= salt_buf2_t[1]; + salt_buf2[2] |= salt_buf2_t[2]; + salt_buf2[3] |= salt_buf2_t[3]; + salt_buf3[0] |= salt_buf3_t[0]; + salt_buf3[1] |= salt_buf3_t[1]; + salt_buf3[2] |= salt_buf3_t[2]; + salt_buf3[3] |= salt_buf3_t[3]; + + append_0x80_4x4_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, salt_pw_salt_len); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0lr; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * put the password after the first salt but before the second salt + */ + + switch_buffer_by_offset_le (t0, t1, t2, t3, salt_len); + + t0[0] |= salt_buf0[0]; + t0[1] |= salt_buf0[1]; + t0[2] |= salt_buf0[2]; + t0[3] |= salt_buf0[3]; + t1[0] |= salt_buf1[0]; + t1[1] |= salt_buf1[1]; + t1[2] |= salt_buf1[2]; + t1[3] |= salt_buf1[3]; + t2[0] |= salt_buf2[0]; + t2[1] |= salt_buf2[1]; + t2[2] |= salt_buf2[2]; + t2[3] |= salt_buf2[3]; + t3[0] |= salt_buf3[0]; + t3[1] |= salt_buf3[1]; + t3[2] = salt_pw_salt_len * 8; + t3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m03800s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + + const u32 salt_pw_salt_len = salt_len + pw_len + salt_len; + + switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_salt_len); + + salt_buf0[0] |= salt_buf0_t[0]; + salt_buf0[1] |= salt_buf0_t[1]; + salt_buf0[2] |= salt_buf0_t[2]; + salt_buf0[3] |= salt_buf0_t[3]; + salt_buf1[0] |= salt_buf1_t[0]; + salt_buf1[1] |= salt_buf1_t[1]; + salt_buf1[2] |= salt_buf1_t[2]; + salt_buf1[3] |= salt_buf1_t[3]; + salt_buf2[0] |= salt_buf2_t[0]; + salt_buf2[1] |= salt_buf2_t[1]; + salt_buf2[2] |= salt_buf2_t[2]; + salt_buf2[3] |= salt_buf2_t[3]; + salt_buf3[0] |= salt_buf3_t[0]; + salt_buf3[1] |= salt_buf3_t[1]; + salt_buf3[2] |= salt_buf3_t[2]; + salt_buf3[3] |= salt_buf3_t[3]; + + append_0x80_4x4_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, salt_pw_salt_len); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0lr; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = w3[2]; + t3[3] = w3[3]; + + /** + * put the password after the first salt but before the second salt + */ + + switch_buffer_by_offset_le (t0, t1, t2, t3, salt_len); + + t0[0] |= salt_buf0[0]; + t0[1] |= salt_buf0[1]; + t0[2] |= salt_buf0[2]; + t0[3] |= salt_buf0[3]; + t1[0] |= salt_buf1[0]; + t1[1] |= salt_buf1[1]; + t1[2] |= salt_buf1[2]; + t1[3] |= salt_buf1[3]; + t2[0] |= salt_buf2[0]; + t2[1] |= salt_buf2[1]; + t2[2] |= salt_buf2[2]; + t2[3] |= salt_buf2[3]; + t3[0] |= salt_buf3[0]; + t3[1] |= salt_buf3[1]; + t3[2] = salt_pw_salt_len * 8; + t3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m03800_a3-pure.cl b/OpenCL/m03800_a3-pure.cl deleted file mode 100644 index 00275dbeb..000000000 --- a/OpenCL/m03800_a3-pure.cl +++ /dev/null @@ -1,186 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector (&ctx, w, pw_len); - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03800_a3.cl b/OpenCL/m03800_a3.cl index fb704eef0..00275dbeb 100644 --- a/OpenCL/m03800_a3.cl +++ b/OpenCL/m03800_a3.cl @@ -11,312 +11,98 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" -void m03800m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m03800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; + u32x w[64] = { 0 }; - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); - const u32 salt_pw_salt_len = salt_len + pw_len + salt_len; + u32x s[64] = { 0 }; - switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_salt_len); + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - salt_buf0[0] |= salt_buf0_t[0]; - salt_buf0[1] |= salt_buf0_t[1]; - salt_buf0[2] |= salt_buf0_t[2]; - salt_buf0[3] |= salt_buf0_t[3]; - salt_buf1[0] |= salt_buf1_t[0]; - salt_buf1[1] |= salt_buf1_t[1]; - salt_buf1[2] |= salt_buf1_t[2]; - salt_buf1[3] |= salt_buf1_t[3]; - salt_buf2[0] |= salt_buf2_t[0]; - salt_buf2[1] |= salt_buf2_t[1]; - salt_buf2[2] |= salt_buf2_t[2]; - salt_buf2[3] |= salt_buf2_t[3]; - salt_buf3[0] |= salt_buf3_t[0]; - salt_buf3[1] |= salt_buf3_t[1]; - salt_buf3[2] |= salt_buf3_t[2]; - salt_buf3[3] |= salt_buf3_t[3]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - append_0x80_4x4_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, salt_pw_salt_len); + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; + w[0] = w0; - t0[0] = w0lr; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + md5_ctx_vector_t ctx; - /** - * put the password after the first salt but before the second salt - */ + md5_init_vector_from_scalar (&ctx, &ctx0); - switch_buffer_by_offset_le (t0, t1, t2, t3, salt_len); + md5_update_vector (&ctx, w, pw_len); - t0[0] |= salt_buf0[0]; - t0[1] |= salt_buf0[1]; - t0[2] |= salt_buf0[2]; - t0[3] |= salt_buf0[3]; - t1[0] |= salt_buf1[0]; - t1[1] |= salt_buf1[1]; - t1[2] |= salt_buf1[2]; - t1[3] |= salt_buf1[3]; - t2[0] |= salt_buf2[0]; - t2[1] |= salt_buf2[1]; - t2[2] |= salt_buf2[2]; - t2[3] |= salt_buf2[3]; - t3[0] |= salt_buf3[0]; - t3[1] |= salt_buf3[1]; - t3[2] = salt_pw_salt_len * 8; - t3[3] = 0; + md5_update_vector (&ctx, s, salt_len); - /** - * md5 - */ + md5_final_vector (&ctx); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m03800s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m03800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; - - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = pw_len + salt_len; - - const u32 salt_pw_salt_len = salt_len + pw_len + salt_len; - - switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_salt_len); - - salt_buf0[0] |= salt_buf0_t[0]; - salt_buf0[1] |= salt_buf0_t[1]; - salt_buf0[2] |= salt_buf0_t[2]; - salt_buf0[3] |= salt_buf0_t[3]; - salt_buf1[0] |= salt_buf1_t[0]; - salt_buf1[1] |= salt_buf1_t[1]; - salt_buf1[2] |= salt_buf1_t[2]; - salt_buf1[3] |= salt_buf1_t[3]; - salt_buf2[0] |= salt_buf2_t[0]; - salt_buf2[1] |= salt_buf2_t[1]; - salt_buf2[2] |= salt_buf2_t[2]; - salt_buf2[3] |= salt_buf2_t[3]; - salt_buf3[0] |= salt_buf3_t[0]; - salt_buf3[1] |= salt_buf3_t[1]; - salt_buf3[2] |= salt_buf3_t[2]; - salt_buf3[3] |= salt_buf3_t[3]; - - append_0x80_4x4_S (salt_buf0, salt_buf1, salt_buf2, salt_buf3, salt_pw_salt_len); + if (gid >= gid_max) return; /** * digest @@ -330,461 +116,71 @@ void m03800s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; + w[0] = w0; - t0[0] = w0lr; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = w3[2]; - t3[3] = w3[3]; + md5_ctx_vector_t ctx; - /** - * put the password after the first salt but before the second salt - */ + md5_init_vector_from_scalar (&ctx, &ctx0); - switch_buffer_by_offset_le (t0, t1, t2, t3, salt_len); + md5_update_vector (&ctx, w, pw_len); - t0[0] |= salt_buf0[0]; - t0[1] |= salt_buf0[1]; - t0[2] |= salt_buf0[2]; - t0[3] |= salt_buf0[3]; - t1[0] |= salt_buf1[0]; - t1[1] |= salt_buf1[1]; - t1[2] |= salt_buf1[2]; - t1[3] |= salt_buf1[3]; - t2[0] |= salt_buf2[0]; - t2[1] |= salt_buf2[1]; - t2[2] |= salt_buf2[2]; - t2[3] |= salt_buf2[3]; - t3[0] |= salt_buf3[0]; - t3[1] |= salt_buf3[1]; - t3[2] = salt_pw_salt_len * 8; - t3[3] = 0; + md5_update_vector (&ctx, s, salt_len); - /** - * md5 - */ + md5_final_vector (&ctx); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, t0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, t3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, t3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, t3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, t3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, t0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, t3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, t0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, t1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, t3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, t1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, t2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, t3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, t3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, t0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, t0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, t3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, t1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, t2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, t0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, t2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m03800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m03800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m03800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m03800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m03800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m03800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - const u32 lid = get_local_id (0); - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m03800s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/OpenCL/m03910_a0-optimized.cl b/OpenCL/m03910_a0-optimized.cl new file mode 100644 index 000000000..e0faae0c1 --- /dev/null +++ b/OpenCL/m03910_a0-optimized.cl @@ -0,0 +1,778 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = salt_buf1[2]; + w3[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = salt_buf1[2]; + w3[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03910_a0-pure.cl b/OpenCL/m03910_a0-pure.cl deleted file mode 100644 index ab08e1a87..000000000 --- a/OpenCL/m03910_a0-pure.cl +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03910_a0.cl b/OpenCL/m03910_a0.cl index e0faae0c1..ab08e1a87 100644 --- a/OpenCL/m03910_a0.cl +++ b/OpenCL/m03910_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,342 +60,113 @@ __kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = salt_buf1[2]; - w3[3] = salt_buf1[3]; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -423,52 +195,6 @@ __kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -481,298 +207,112 @@ __kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = salt_buf1[2]; - w3[3] = salt_buf1[3]; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03910_a1-optimized.cl b/OpenCL/m03910_a1-optimized.cl new file mode 100644 index 000000000..803aaa8d4 --- /dev/null +++ b/OpenCL/m03910_a1-optimized.cl @@ -0,0 +1,894 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = salt_buf1[2]; + w3[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = salt_buf1[2]; + w3[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m03910_a1-pure.cl b/OpenCL/m03910_a1-pure.cl deleted file mode 100644 index c4186f01e..000000000 --- a/OpenCL/m03910_a1-pure.cl +++ /dev/null @@ -1,294 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03910_a1.cl b/OpenCL/m03910_a1.cl index 803aaa8d4..c4186f01e 100644 --- a/OpenCL/m03910_a1.cl +++ b/OpenCL/m03910_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,401 +58,102 @@ __kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; + const u32 salt_len = 32; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - const u32 pw_l_len = pws[gid].pw_len; + u32x s[8] = { 0 }; - /** - * salt - */ + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + md5_ctx_t ctx0; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = salt_buf1[2]; - w3[3] = salt_buf1[3]; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -480,52 +182,6 @@ __kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -538,357 +194,101 @@ __kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_transform (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = salt_buf1[2]; - w3[3] = salt_buf1[3]; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m03910_a3-optimized.cl b/OpenCL/m03910_a3-optimized.cl new file mode 100644 index 000000000..6ecfa2579 --- /dev/null +++ b/OpenCL/m03910_a3-optimized.cl @@ -0,0 +1,1144 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m03910m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = salt_buf1[2]; + w3_t[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m03910s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = salt_buf1[2]; + w3_t[3] = salt_buf1[3]; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + const u32x r_a = a + MD5M_A; + const u32x r_b = b + MD5M_B; + const u32x r_c = c + MD5M_C; + const u32x r_d = d + MD5M_D; + + const u32x r_00 = 0x80; + const u32x r_14 = 64 * 8; + + a = r_a; + b = r_b; + c = r_c; + d = r_d; + + MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); + MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); + MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); + MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); + MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); + + MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); + MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); + MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); + MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); + MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); + + MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); + MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); + MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); + MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); + MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); + MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); + MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); + + if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; + + MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); + MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); + MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); + + a += r_a; + b += r_b; + c += r_c; + d += r_d; + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m03910_a3-pure.cl b/OpenCL/m03910_a3-pure.cl deleted file mode 100644 index 48e7ae215..000000000 --- a/OpenCL/m03910_a3-pure.cl +++ /dev/null @@ -1,328 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = 32; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[8] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - ctx.w2[0] = s[0]; - ctx.w2[1] = s[1]; - ctx.w2[2] = s[2]; - ctx.w2[3] = s[3]; - ctx.w3[0] = s[4]; - ctx.w3[1] = s[5]; - ctx.w3[2] = s[6]; - ctx.w3[3] = s[7]; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - ctx.w0[0] = 0x80; - ctx.w0[1] = 0; - ctx.w0[2] = 0; - ctx.w0[3] = 0; - ctx.w1[0] = 0; - ctx.w1[1] = 0; - ctx.w1[2] = 0; - ctx.w1[3] = 0; - ctx.w2[0] = 0; - ctx.w2[1] = 0; - ctx.w2[2] = 0; - ctx.w2[3] = 0; - ctx.w3[0] = 0; - ctx.w3[1] = 0; - ctx.w3[2] = 64 * 8; - ctx.w3[3] = 0; - - md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m03910_a3.cl b/OpenCL/m03910_a3.cl index 6ecfa2579..48e7ae215 100644 --- a/OpenCL/m03910_a3.cl +++ b/OpenCL/m03910_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m03910m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m03910_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,341 +33,144 @@ void m03910m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = salt_buf1[2]; - w3_t[3] = salt_buf1[3]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m03910s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m03910_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -374,34 +178,26 @@ void m03910s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf_pc[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf_pc[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf_pc[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf_pc[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf_pc[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf_pc[5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf_pc[6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf_pc[7]; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -415,730 +211,118 @@ void m03910s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = 32; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[8] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf_pc[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + ctx.w2[0] = s[0]; + ctx.w2[1] = s[1]; + ctx.w2[2] = s[2]; + ctx.w2[3] = s[3]; + ctx.w3[0] = s[4]; + ctx.w3[1] = s[5]; + ctx.w3[2] = s[6]; + ctx.w3[3] = s[7]; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = salt_buf1[2]; - w3_t[3] = salt_buf1[3]; + ctx.w0[0] = 0x80; + ctx.w0[1] = 0; + ctx.w0[2] = 0; + ctx.w0[3] = 0; + ctx.w1[0] = 0; + ctx.w1[1] = 0; + ctx.w1[2] = 0; + ctx.w1[3] = 0; + ctx.w2[0] = 0; + ctx.w2[1] = 0; + ctx.w2[2] = 0; + ctx.w2[3] = 0; + ctx.w3[0] = 0; + ctx.w3[1] = 0; + ctx.w3[2] = 64 * 8; + ctx.w3[3] = 0; - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_transform_vector (ctx.w0, ctx.w1, ctx.w2, ctx.w3, ctx.h); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - const u32x r_a = a + MD5M_A; - const u32x r_b = b + MD5M_B; - const u32x r_c = c + MD5M_C; - const u32x r_d = d + MD5M_D; - - const u32x r_00 = 0x80; - const u32x r_14 = 64 * 8; - - a = r_a; - b = r_b; - c = r_c; - d = r_d; - - MD5_STEP (MD5_Fo, a, b, c, d, r_00, MD5C00, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C01, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C02, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C03, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C04, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C05, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C06, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C07, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C08, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C09, MD5S01); - MD5_STEP0(MD5_Fo, c, d, a, b, MD5C0a, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0b, MD5S03); - MD5_STEP0(MD5_Fo, a, b, c, d, MD5C0c, MD5S00); - MD5_STEP0(MD5_Fo, d, a, b, c, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, r_14, MD5C0e, MD5S02); - MD5_STEP0(MD5_Fo, b, c, d, a, MD5C0f, MD5S03); - - MD5_STEP0(MD5_Go, a, b, c, d, MD5C10, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C11, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, r_00, MD5C13, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C14, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C15, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C16, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C17, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, r_14, MD5C19, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1a, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1b, MD5S13); - MD5_STEP0(MD5_Go, a, b, c, d, MD5C1c, MD5S10); - MD5_STEP0(MD5_Go, d, a, b, c, MD5C1d, MD5S11); - MD5_STEP0(MD5_Go, c, d, a, b, MD5C1e, MD5S12); - MD5_STEP0(MD5_Go, b, c, d, a, MD5C1f, MD5S13); - - MD5_STEP0(MD5_H , a, b, c, d, MD5C20, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C21, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, r_14, MD5C23, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C24, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C25, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C26, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C27, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, r_00, MD5C29, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2a, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2b, MD5S23); - MD5_STEP0(MD5_H , a, b, c, d, MD5C2c, MD5S20); - MD5_STEP0(MD5_H , d, a, b, c, MD5C2d, MD5S21); - MD5_STEP0(MD5_H , c, d, a, b, MD5C2e, MD5S22); - MD5_STEP0(MD5_H , b, c, d, a, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, r_00, MD5C30, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, r_14, MD5C32, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C33, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C34, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C35, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C36, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C37, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C38, MD5S30); - MD5_STEP0(MD5_I , d, a, b, c, MD5C39, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3a, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3b, MD5S33); - MD5_STEP0(MD5_I , a, b, c, d, MD5C3c, MD5S30); - - if (MATCHES_NONE_VS ((a + r_a), search[0])) continue; - - MD5_STEP0(MD5_I , d, a, b, c, MD5C3d, MD5S31); - MD5_STEP0(MD5_I , c, d, a, b, MD5C3e, MD5S32); - MD5_STEP0(MD5_I , b, c, d, a, MD5C3f, MD5S33); - - a += r_a; - b += r_b; - c += r_c; - d += r_d; - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m03910_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03910_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03910_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03910_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03910_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m03910_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m03910s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m04010_a0-optimized.cl b/OpenCL/m04010_a0-optimized.cl new file mode 100644 index 000000000..9d008c854 --- /dev/null +++ b/OpenCL/m04010_a0-optimized.cl @@ -0,0 +1,720 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = salt_len + out_len; + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = 32 + salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * prepend salt + */ + + const u32x out_salt_len = salt_len + out_len; + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = out_salt_len * 8; + w3[3] = 0; + + append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04010_a0-pure.cl b/OpenCL/m04010_a0-pure.cl deleted file mode 100644 index 454997290..000000000 --- a/OpenCL/m04010_a0-pure.cl +++ /dev/null @@ -1,272 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx1 = ctx0; - - md5_update (&ctx1, w, pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx1 = ctx0; - - md5_update (&ctx1, w, pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04010_a0.cl b/OpenCL/m04010_a0.cl index 9d008c854..454997290 100644 --- a/OpenCL/m04010_a0.cl +++ b/OpenCL/m04010_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,171 +60,48 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 pw_salt_len = 32 + salt_len; + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx1 = ctx0; - /** - * prepend salt - */ + md5_update (&ctx1, w, pw_len); - const u32x out_salt_len = salt_len + out_len; + md5_final (&ctx1); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -241,7 +119,7 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -250,122 +128,22 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -394,54 +172,6 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 pw_salt_len = 32 + salt_len; - /** * digest */ @@ -454,127 +184,52 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx1 = ctx0; - /** - * prepend salt - */ + md5_update (&ctx1, w, pw_len); - const u32x out_salt_len = salt_len + out_len; + md5_final (&ctx1); - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = out_salt_len * 8; - w3[3] = 0; - - append_0x80_4x4_VV (w0, w1, w2, w3, out_salt_len); - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -592,7 +247,7 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -601,120 +256,17 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = pw_salt_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04010_a1-optimized.cl b/OpenCL/m04010_a1-optimized.cl new file mode 100644 index 000000000..120b4778a --- /dev/null +++ b/OpenCL/m04010_a1-optimized.cl @@ -0,0 +1,834 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 final_len = salt_len + 32; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 final_len = salt_len + 32; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04010_a1-pure.cl b/OpenCL/m04010_a1-pure.cl deleted file mode 100644 index 98f082c00..000000000 --- a/OpenCL/m04010_a1-pure.cl +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_ctx_t ctx0t = ctx0; - - md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0t; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - md5_ctx_t ctx0t = ctx0; - - md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0t; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04010_a1.cl b/OpenCL/m04010_a1.cl index 120b4778a..98f082c00 100644 --- a/OpenCL/m04010_a1.cl +++ b/OpenCL/m04010_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,229 +58,37 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; + md5_ctx_t ctx0; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + md5_init (&ctx0); - const u32 pw_l_len = pws[gid].pw_len; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - /** - * salt - */ + md5_ctx_t ctx0t = ctx0; - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 final_len = salt_len + 32; + md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0t; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -297,7 +106,7 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -306,122 +115,22 @@ __kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -450,54 +159,6 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 final_len = salt_len + 32; - /** * digest */ @@ -510,185 +171,41 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + md5_ctx_t ctx0t = ctx0; + + md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0t; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * prepend salt - */ - - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -706,7 +223,7 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -715,120 +232,17 @@ __kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04010_a3-optimized.cl b/OpenCL/m04010_a3-optimized.cl new file mode 100644 index 000000000..54c2faca8 --- /dev/null +++ b/OpenCL/m04010_a3-optimized.cl @@ -0,0 +1,1130 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m04010m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + const u32 final_len = salt_len + 32; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = t3[2]; + w3_t[3] = t3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = final_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m04010s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + const u32 final_len = salt_len + 32; + + /** + * prepend salt + */ + + const u32 w0l = w0[0]; + + switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + u32x t0[4]; + u32x t1[4]; + u32x t2[4]; + u32x t3[4]; + + t0[0] = w0[0]; + t0[1] = w0[1]; + t0[2] = w0[2]; + t0[3] = w0[3]; + t1[0] = w1[0]; + t1[1] = w1[1]; + t1[2] = w1[2]; + t1[3] = w1[3]; + t2[0] = w2[0]; + t2[1] = w2[1]; + t2[2] = w2[2]; + t2[3] = w2[3]; + t3[0] = w3[0]; + t3[1] = w3[1]; + t3[2] = pw_salt_len * 8; + t3[3] = 0; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = t0[0]; + w0_t[1] = t0[1]; + w0_t[2] = t0[2]; + w0_t[3] = t0[3]; + w1_t[0] = t1[0]; + w1_t[1] = t1[1]; + w1_t[2] = t1[2]; + w1_t[3] = t1[3]; + w2_t[0] = t2[0]; + w2_t[1] = t2[1]; + w2_t[2] = t2[2]; + w2_t[3] = t2[3]; + w3_t[0] = t3[0]; + w3_t[1] = t3[1]; + w3_t[2] = t3[2]; + w3_t[3] = t3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = final_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m04010_a3-pure.cl b/OpenCL/m04010_a3-pure.cl deleted file mode 100644 index 4822b707b..000000000 --- a/OpenCL/m04010_a3-pure.cl +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx1; - - md5_init_vector_from_scalar (&ctx1, &ctx0); - - md5_update_vector (&ctx1, w, pw_len); - - md5_final_vector (&ctx1); - - const u32x a = ctx1.h[0]; - const u32x b = ctx1.h[1]; - const u32x c = ctx1.h[2]; - const u32x d = ctx1.h[3]; - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx1; - - md5_init_vector_from_scalar (&ctx1, &ctx0); - - md5_update_vector (&ctx1, w, pw_len); - - md5_final_vector (&ctx1); - - const u32x a = ctx1.h[0]; - const u32x b = ctx1.h[1]; - const u32x c = ctx1.h[2]; - const u32x d = ctx1.h[3]; - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04010_a3.cl b/OpenCL/m04010_a3.cl index 54c2faca8..4822b707b 100644 --- a/OpenCL/m04010_a3.cl +++ b/OpenCL/m04010_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m04010m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04010_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,334 +33,125 @@ void m04010m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } - const u32 pw_salt_len = pw_len + salt_len; - const u32 final_len = salt_len + 32; + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** - * prepend salt + * base */ - const u32 w0l = w0[0]; + const u32 pw_len = pws[gid].pw_len; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + const u32 pw_lenv = ceil ((float) pw_len / 4); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + u32x w[64] = { 0 }; - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0lr; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md5_ctx_vector_t ctx1; - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = t3[2]; - w3_t[3] = t3[3]; + md5_init_vector_from_scalar (&ctx1, &ctx0); - /** - * md5 - */ + md5_update_vector (&ctx1, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final_vector (&ctx1); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x a = ctx1.h[0]; + const u32x b = ctx1.h[1]; + const u32x c = ctx1.h[2]; + const u32x d = ctx1.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_ctx_vector_t ctx; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - /** - * prepend salt - */ + md5_final_vector (&ctx); - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w3_t[2] = final_len * 8; - w3_t[3] = 0; - - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m04010s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04010_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -367,6 +159,26 @@ void m04010s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -381,750 +193,98 @@ void m04010s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - const u32 salt_len = salt_bufs[salt_pos].salt_len; + u32x w[64] = { 0 }; - const u32 pw_salt_len = pw_len + salt_len; - const u32 final_len = salt_len + 32; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * prepend salt - */ + barrier (CLK_GLOBAL_MEM_FENCE); + } - const u32 w0l = w0[0]; + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (w0, w1, w2, w3, salt_len); + md5_init (&ctx0); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; - - u32x t0[4]; - u32x t1[4]; - u32x t2[4]; - u32x t3[4]; - - t0[0] = w0[0]; - t0[1] = w0[1]; - t0[2] = w0[2]; - t0[3] = w0[3]; - t1[0] = w1[0]; - t1[1] = w1[1]; - t1[2] = w1[2]; - t1[3] = w1[3]; - t2[0] = w2[0]; - t2[1] = w2[1]; - t2[2] = w2[2]; - t2[3] = w2[3]; - t3[0] = w3[0]; - t3[1] = w3[1]; - t3[2] = pw_salt_len * 8; - t3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ + u32x w0l = w[0]; + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - overwrite_at_le_4x4 (t0, t1, t2, t3, w0lr, salt_len); + w[0] = w0lr; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + md5_ctx_vector_t ctx1; - w0_t[0] = t0[0]; - w0_t[1] = t0[1]; - w0_t[2] = t0[2]; - w0_t[3] = t0[3]; - w1_t[0] = t1[0]; - w1_t[1] = t1[1]; - w1_t[2] = t1[2]; - w1_t[3] = t1[3]; - w2_t[0] = t2[0]; - w2_t[1] = t2[1]; - w2_t[2] = t2[2]; - w2_t[3] = t2[3]; - w3_t[0] = t3[0]; - w3_t[1] = t3[1]; - w3_t[2] = t3[2]; - w3_t[3] = t3[3]; + md5_init_vector_from_scalar (&ctx1, &ctx0); - /** - * md5 - */ + md5_update_vector (&ctx1, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final_vector (&ctx1); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x a = ctx1.h[0]; + const u32x b = ctx1.h[1]; + const u32x c = ctx1.h[2]; + const u32x d = ctx1.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_ctx_vector_t ctx; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + md5_init_vector_from_scalar (&ctx, &ctx0); - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - /** - * prepend salt - */ + md5_final_vector (&ctx); - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - w3_t[2] = final_len * 8; - w3_t[3] = 0; - - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; - - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m04010_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04010_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04010_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04010m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04010_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04010_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04010_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m04010s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m04110_a0-optimized.cl b/OpenCL/m04110_a0-optimized.cl new file mode 100644 index 000000000..0f095ee49 --- /dev/null +++ b/OpenCL/m04110_a0-optimized.cl @@ -0,0 +1,810 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 final_len = salt_len + 32; + + // salt version 2: with 0x80 appended + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0_t[0]; + s0[1] = salt_buf0_t[1]; + s0[2] = salt_buf0_t[2]; + s0[3] = salt_buf0_t[3]; + s1[0] = salt_buf1_t[0]; + s1[1] = salt_buf1_t[1]; + s1[2] = salt_buf1_t[2]; + s1[3] = salt_buf1_t[3]; + s2[0] = salt_buf2_t[0]; + s2[1] = salt_buf2_t[1]; + s2[2] = salt_buf2_t[2]; + s2[3] = salt_buf2_t[3]; + s3[0] = salt_buf3_t[0]; + s3[1] = salt_buf3_t[1]; + s3[2] = salt_buf3_t[2]; + s3[3] = salt_buf3_t[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 final_len = salt_len + 32; + + // salt version 2: with 0x80 appended + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0_t[0]; + s0[1] = salt_buf0_t[1]; + s0[2] = salt_buf0_t[2]; + s0[3] = salt_buf0_t[3]; + s1[0] = salt_buf1_t[0]; + s1[1] = salt_buf1_t[1]; + s1[2] = salt_buf1_t[2]; + s1[3] = salt_buf1_t[3]; + s2[0] = salt_buf2_t[0]; + s2[1] = salt_buf2_t[1]; + s2[2] = salt_buf2_t[2]; + s2[3] = salt_buf2_t[3]; + s3[0] = salt_buf3_t[0]; + s3[1] = salt_buf3_t[1]; + s3[2] = salt_buf3_t[2]; + s3[3] = salt_buf3_t[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + + const u32x pw_salt_len = out_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04110_a0-pure.cl b/OpenCL/m04110_a0-pure.cl deleted file mode 100644 index 4bcb4def6..000000000 --- a/OpenCL/m04110_a0-pure.cl +++ /dev/null @@ -1,306 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx1; - - md5_init (&ctx1); - - md5_update (&ctx1, w, pw_len); - - md5_update (&ctx1, s, salt_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx1; - - md5_init (&ctx1); - - md5_update (&ctx1, w, pw_len); - - md5_update (&ctx1, s, salt_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04110_a0.cl b/OpenCL/m04110_a0.cl index 0f095ee49..4bcb4def6 100644 --- a/OpenCL/m04110_a0.cl +++ b/OpenCL/m04110_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,216 +60,65 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 final_len = salt_len + 32; + barrier (CLK_GLOBAL_MEM_FENCE); + } - // salt version 2: with 0x80 appended + const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + u32 s[64] = { 0 }; - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx1; - /** - * append salt - */ + md5_init (&ctx1); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_update (&ctx1, w, pw_len); - s0[0] = salt_buf0_t[0]; - s0[1] = salt_buf0_t[1]; - s0[2] = salt_buf0_t[2]; - s0[3] = salt_buf0_t[3]; - s1[0] = salt_buf1_t[0]; - s1[1] = salt_buf1_t[1]; - s1[2] = salt_buf1_t[2]; - s1[3] = salt_buf1_t[3]; - s2[0] = salt_buf2_t[0]; - s2[1] = salt_buf2_t[1]; - s2[2] = salt_buf2_t[2]; - s2[3] = salt_buf2_t[3]; - s3[0] = salt_buf3_t[0]; - s3[1] = salt_buf3_t[1]; - s3[2] = salt_buf3_t[2]; - s3[3] = salt_buf3_t[3]; + md5_update (&ctx1, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + md5_final (&ctx1); - const u32x pw_salt_len = out_len + salt_len; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -286,7 +136,7 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -295,122 +145,22 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -439,79 +189,6 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 final_len = salt_len + 32; - - // salt version 2: with 0x80 appended - - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; - - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; - - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); - /** * digest */ @@ -524,147 +201,69 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx1; - /** - * append salt - */ + md5_init (&ctx1); - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; + md5_update (&ctx1, w, pw_len); - s0[0] = salt_buf0_t[0]; - s0[1] = salt_buf0_t[1]; - s0[2] = salt_buf0_t[2]; - s0[3] = salt_buf0_t[3]; - s1[0] = salt_buf1_t[0]; - s1[1] = salt_buf1_t[1]; - s1[2] = salt_buf1_t[2]; - s1[3] = salt_buf1_t[3]; - s2[0] = salt_buf2_t[0]; - s2[1] = salt_buf2_t[1]; - s2[2] = salt_buf2_t[2]; - s2[3] = salt_buf2_t[3]; - s3[0] = salt_buf3_t[0]; - s3[1] = salt_buf3_t[1]; - s3[2] = salt_buf3_t[2]; - s3[3] = salt_buf3_t[3]; + md5_update (&ctx1, s, salt_len); - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, out_len); + md5_final (&ctx1); - const u32x pw_salt_len = out_len + salt_len; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -682,7 +281,7 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -691,120 +290,17 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04110_a1-optimized.cl b/OpenCL/m04110_a1-optimized.cl new file mode 100644 index 000000000..7e9a98608 --- /dev/null +++ b/OpenCL/m04110_a1-optimized.cl @@ -0,0 +1,928 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 final_len = salt_len + 32; + + // salt version 2: with 0x80 appended + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0_t[0]; + s0[1] = salt_buf0_t[1]; + s0[2] = salt_buf0_t[2]; + s0[3] = salt_buf0_t[3]; + s1[0] = salt_buf1_t[0]; + s1[1] = salt_buf1_t[1]; + s1[2] = salt_buf1_t[2]; + s1[3] = salt_buf1_t[3]; + s2[0] = salt_buf2_t[0]; + s2[1] = salt_buf2_t[1]; + s2[2] = salt_buf2_t[2]; + s2[3] = salt_buf2_t[3]; + s3[0] = salt_buf3_t[0]; + s3[1] = salt_buf3_t[1]; + s3[2] = salt_buf3_t[2]; + s3[3] = salt_buf3_t[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 final_len = salt_len + 32; + + // salt version 2: with 0x80 appended + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * append salt + */ + + u32x s0[4]; + u32x s1[4]; + u32x s2[4]; + u32x s3[4]; + + s0[0] = salt_buf0_t[0]; + s0[1] = salt_buf0_t[1]; + s0[2] = salt_buf0_t[2]; + s0[3] = salt_buf0_t[3]; + s1[0] = salt_buf1_t[0]; + s1[1] = salt_buf1_t[1]; + s1[2] = salt_buf1_t[2]; + s1[3] = salt_buf1_t[3]; + s2[0] = salt_buf2_t[0]; + s2[1] = salt_buf2_t[1]; + s2[2] = salt_buf2_t[2]; + s2[3] = salt_buf2_t[3]; + s3[0] = salt_buf3_t[0]; + s3[1] = salt_buf3_t[1]; + s3[2] = salt_buf3_t[2]; + s3[3] = salt_buf3_t[3]; + + switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); + + const u32x pw_salt_len = pw_len + salt_len; + + w0[0] |= s0[0]; + w0[1] |= s0[1]; + w0[2] |= s0[2]; + w0[3] |= s0[3]; + w1[0] |= s1[0]; + w1[1] |= s1[1]; + w1[2] |= s1[2]; + w1[3] |= s1[3]; + w2[0] |= s2[0]; + w2[1] |= s2[1]; + w2[2] |= s2[2]; + w2[3] |= s2[3]; + w3[0] |= s3[0]; + w3[1] |= s3[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0x80; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + + w3[2] = final_len * 8; + w3[3] = 0; + + w0[0] |= salt_buf0[0]; + w0[1] |= salt_buf0[1]; + w0[2] |= salt_buf0[2]; + w0[3] |= salt_buf0[3]; + w1[0] |= salt_buf1[0]; + w1[1] |= salt_buf1[1]; + w1[2] |= salt_buf1[2]; + w1[3] |= salt_buf1[3]; + w2[0] |= salt_buf2[0]; + w2[1] |= salt_buf2[1]; + w2[2] |= salt_buf2[2]; + w2[3] |= salt_buf2[3]; + w3[0] |= salt_buf3[0]; + w3[1] |= salt_buf3[1]; + w3[2] |= salt_buf3[2]; + w3[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04110_a1-pure.cl b/OpenCL/m04110_a1-pure.cl deleted file mode 100644 index 57684942a..000000000 --- a/OpenCL/m04110_a1-pure.cl +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - md5_ctx_t ctx0t; - - md5_init (&ctx0t); - - md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0t; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx1, s, salt_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, s, salt_len); - - md5_ctx_t ctx0t; - - md5_init (&ctx0t); - - md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0t; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_update (&ctx1, s, salt_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_t ctx = ctx0; - - md5_update_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04110_a1.cl b/OpenCL/m04110_a1.cl index 7e9a98608..57684942a 100644 --- a/OpenCL/m04110_a1.cl +++ b/OpenCL/m04110_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,276 +58,54 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - const u32 pw_l_len = pws[gid].pw_len; + u32 s[64] = { 0 }; - /** - * salt - */ + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + md5_ctx_t ctx0; - const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 final_len = salt_len + 32; + md5_init (&ctx0); - // salt version 2: with 0x80 appended + md5_update (&ctx0, s, salt_len); - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; + md5_ctx_t ctx0t; - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; + md5_init (&ctx0t); - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0t; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx1, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx1); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0_t[0]; - s0[1] = salt_buf0_t[1]; - s0[2] = salt_buf0_t[2]; - s0[3] = salt_buf0_t[3]; - s1[0] = salt_buf1_t[0]; - s1[1] = salt_buf1_t[1]; - s1[2] = salt_buf1_t[2]; - s1[3] = salt_buf1_t[3]; - s2[0] = salt_buf2_t[0]; - s2[1] = salt_buf2_t[1]; - s2[2] = salt_buf2_t[2]; - s2[3] = salt_buf2_t[3]; - s3[0] = salt_buf3_t[0]; - s3[1] = salt_buf3_t[1]; - s3[2] = salt_buf3_t[2]; - s3[3] = salt_buf3_t[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -344,7 +123,7 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -353,122 +132,22 @@ __kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -497,79 +176,6 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 final_len = salt_len + 32; - - // salt version 2: with 0x80 appended - - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; - - salt_buf0_t[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0_t[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0_t[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0_t[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1_t[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1_t[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1_t[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1_t[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2_t[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2_t[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2_t[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2_t[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3_t[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3_t[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3_t[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3_t[3] = salt_bufs[salt_pos].salt_buf[15]; - - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); - /** * digest */ @@ -582,207 +188,58 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update (&ctx0, s, salt_len); + + md5_ctx_t ctx0t; + + md5_init (&ctx0t); + + md5_update_global (&ctx0t, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0t; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_update (&ctx1, s, salt_len); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + md5_final (&ctx1); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; - - /** - * append salt - */ - - u32x s0[4]; - u32x s1[4]; - u32x s2[4]; - u32x s3[4]; - - s0[0] = salt_buf0_t[0]; - s0[1] = salt_buf0_t[1]; - s0[2] = salt_buf0_t[2]; - s0[3] = salt_buf0_t[3]; - s1[0] = salt_buf1_t[0]; - s1[1] = salt_buf1_t[1]; - s1[2] = salt_buf1_t[2]; - s1[3] = salt_buf1_t[3]; - s2[0] = salt_buf2_t[0]; - s2[1] = salt_buf2_t[1]; - s2[2] = salt_buf2_t[2]; - s2[3] = salt_buf2_t[3]; - s3[0] = salt_buf3_t[0]; - s3[1] = salt_buf3_t[1]; - s3[2] = salt_buf3_t[2]; - s3[3] = salt_buf3_t[3]; - - switch_buffer_by_offset_le_VV (s0, s1, s2, s3, pw_len); - - const u32x pw_salt_len = pw_len + salt_len; - - w0[0] |= s0[0]; - w0[1] |= s0[1]; - w0[2] |= s0[2]; - w0[3] |= s0[3]; - w1[0] |= s1[0]; - w1[1] |= s1[1]; - w1[2] |= s1[2]; - w1[3] |= s1[3]; - w2[0] |= s2[0]; - w2[1] |= s2[1]; - w2[2] |= s2[2]; - w2[3] |= s2[3]; - w3[0] |= s3[0]; - w3[1] |= s3[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; - - /** - * md5 - */ - - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 | uint_to_hex_lower8 ((a >> 8) & 255) << 16; @@ -800,7 +257,7 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule | uint_to_hex_lower8 ((d >> 8) & 255) << 16; w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0x80; + w2[0] = 0; w2[1] = 0; w2[2] = 0; w2[3] = 0; @@ -809,120 +266,17 @@ __kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rule w3[2] = 0; w3[3] = 0; - /** - * prepend salt - */ + md5_ctx_t ctx = ctx0; - switch_buffer_by_offset_le (w0, w1, w2, w3, salt_len); + md5_update_64 (&ctx, w0, w1, w2, w3, 32); - w3[2] = final_len * 8; - w3[3] = 0; + md5_final (&ctx); - w0[0] |= salt_buf0[0]; - w0[1] |= salt_buf0[1]; - w0[2] |= salt_buf0[2]; - w0[3] |= salt_buf0[3]; - w1[0] |= salt_buf1[0]; - w1[1] |= salt_buf1[1]; - w1[2] |= salt_buf1[2]; - w1[3] |= salt_buf1[3]; - w2[0] |= salt_buf2[0]; - w2[1] |= salt_buf2[1]; - w2[2] |= salt_buf2[2]; - w2[3] |= salt_buf2[3]; - w3[0] |= salt_buf3[0]; - w3[1] |= salt_buf3[1]; - w3[2] |= salt_buf3[2]; - w3[3] |= salt_buf3[3]; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04110_a3-optimized.cl b/OpenCL/m04110_a3-optimized.cl new file mode 100644 index 000000000..9cda9fda4 --- /dev/null +++ b/OpenCL/m04110_a3-optimized.cl @@ -0,0 +1,1130 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m04110m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + const u32 final_len = salt_len + 32; + + /** + * append salt + */ + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_buf0[0]; + salt_buf0_t[1] = salt_buf0[1]; + salt_buf0_t[2] = salt_buf0[2]; + salt_buf0_t[3] = salt_buf0[3]; + salt_buf1_t[0] = salt_buf1[0]; + salt_buf1_t[1] = salt_buf1[1]; + salt_buf1_t[2] = salt_buf1[2]; + salt_buf1_t[3] = salt_buf1[3]; + salt_buf2_t[0] = salt_buf2[0]; + salt_buf2_t[1] = salt_buf2[1]; + salt_buf2_t[2] = salt_buf2[2]; + salt_buf2_t[3] = salt_buf2[3]; + salt_buf3_t[0] = salt_buf3[0]; + salt_buf3_t[1] = salt_buf3[1]; + salt_buf3_t[2] = salt_buf3[2]; + salt_buf3_t[3] = salt_buf3[3]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_len); + + w0[0] |= salt_buf0_t[0]; + w0[1] |= salt_buf0_t[1]; + w0[2] |= salt_buf0_t[2]; + w0[3] |= salt_buf0_t[3]; + w1[0] |= salt_buf1_t[0]; + w1[1] |= salt_buf1_t[1]; + w1[2] |= salt_buf1_t[2]; + w1[3] |= salt_buf1_t[3]; + w2[0] |= salt_buf2_t[0]; + w2[1] |= salt_buf2_t[1]; + w2[2] |= salt_buf2_t[2]; + w2[3] |= salt_buf2_t[3]; + w3[0] |= salt_buf3_t[0]; + w3[1] |= salt_buf3_t[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = final_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m04110s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; + salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; + salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; + salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; + salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; + salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; + salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; + salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; + salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; + salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; + salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 pw_salt_len = pw_len + salt_len; + const u32 final_len = salt_len + 32; + + /** + * append salt + */ + + u32 salt_buf0_t[4]; + u32 salt_buf1_t[4]; + u32 salt_buf2_t[4]; + u32 salt_buf3_t[4]; + + salt_buf0_t[0] = salt_buf0[0]; + salt_buf0_t[1] = salt_buf0[1]; + salt_buf0_t[2] = salt_buf0[2]; + salt_buf0_t[3] = salt_buf0[3]; + salt_buf1_t[0] = salt_buf1[0]; + salt_buf1_t[1] = salt_buf1[1]; + salt_buf1_t[2] = salt_buf1[2]; + salt_buf1_t[3] = salt_buf1[3]; + salt_buf2_t[0] = salt_buf2[0]; + salt_buf2_t[1] = salt_buf2[1]; + salt_buf2_t[2] = salt_buf2[2]; + salt_buf2_t[3] = salt_buf2[3]; + salt_buf3_t[0] = salt_buf3[0]; + salt_buf3_t[1] = salt_buf3[1]; + salt_buf3_t[2] = salt_buf3[2]; + salt_buf3_t[3] = salt_buf3[3]; + + append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + + switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_len); + + w0[0] |= salt_buf0_t[0]; + w0[1] |= salt_buf0_t[1]; + w0[2] |= salt_buf0_t[2]; + w0[3] |= salt_buf0_t[3]; + w1[0] |= salt_buf1_t[0]; + w1[1] |= salt_buf1_t[1]; + w1[2] |= salt_buf1_t[2]; + w1[3] |= salt_buf1_t[3]; + w2[0] |= salt_buf2_t[0]; + w2[1] |= salt_buf2_t[1]; + w2[2] |= salt_buf2_t[2]; + w2[3] |= salt_buf2_t[3]; + w3[0] |= salt_buf3_t[0]; + w3[1] |= salt_buf3_t[1]; + w3[2] = pw_salt_len * 8; + w3[3] = 0; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2_t[0] = 0x80; + w2_t[1] = 0; + w2_t[2] = 0; + w2_t[3] = 0; + w3_t[0] = 0; + w3_t[1] = 0; + w3_t[2] = 0; + w3_t[3] = 0; + + /** + * prepend salt + */ + + switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + + w3_t[2] = final_len * 8; + w3_t[3] = 0; + + w0_t[0] |= salt_buf0[0]; + w0_t[1] |= salt_buf0[1]; + w0_t[2] |= salt_buf0[2]; + w0_t[3] |= salt_buf0[3]; + w1_t[0] |= salt_buf1[0]; + w1_t[1] |= salt_buf1[1]; + w1_t[2] |= salt_buf1[2]; + w1_t[3] |= salt_buf1[3]; + w2_t[0] |= salt_buf2[0]; + w2_t[1] |= salt_buf2[1]; + w2_t[2] |= salt_buf2[2]; + w2_t[3] |= salt_buf2[3]; + w3_t[0] |= salt_buf3[0]; + w3_t[1] |= salt_buf3[1]; + w3_t[2] |= salt_buf3[2]; + w3_t[3] |= salt_buf3[3]; + + /** + * md5 + */ + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m04110_a3-pure.cl b/OpenCL/m04110_a3-pure.cl deleted file mode 100644 index f466a3178..000000000 --- a/OpenCL/m04110_a3-pure.cl +++ /dev/null @@ -1,320 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx1; - - md5_init_vector (&ctx1); - - md5_update_vector (&ctx1, w, pw_len); - - md5_update_vector (&ctx1, s, salt_len); - - md5_final_vector (&ctx1); - - const u32x a = ctx1.h[0]; - const u32x b = ctx1.h[1]; - const u32x c = ctx1.h[2]; - const u32x d = ctx1.h[3]; - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0lr = w0l | w0r; - - w[0] = w0lr; - - md5_ctx_vector_t ctx1; - - md5_init_vector (&ctx1); - - md5_update_vector (&ctx1, w, pw_len); - - md5_update_vector (&ctx1, s, salt_len); - - md5_final_vector (&ctx1); - - const u32x a = ctx1.h[0]; - const u32x b = ctx1.h[1]; - const u32x c = ctx1.h[2]; - const u32x d = ctx1.h[3]; - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - md5_ctx_vector_t ctx; - - md5_init_vector_from_scalar (&ctx, &ctx0); - - md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04110_a3.cl b/OpenCL/m04110_a3.cl index 9cda9fda4..f466a3178 100644 --- a/OpenCL/m04110_a3.cl +++ b/OpenCL/m04110_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_lower8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m04110m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04110_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,334 +33,140 @@ void m04110m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; - const u32 final_len = salt_len + 32; + const u32 salt_lenv = ceil ((float) salt_len / 4); - /** - * append salt - */ + u32x s[64] = { 0 }; - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - salt_buf0_t[0] = salt_buf0[0]; - salt_buf0_t[1] = salt_buf0[1]; - salt_buf0_t[2] = salt_buf0[2]; - salt_buf0_t[3] = salt_buf0[3]; - salt_buf1_t[0] = salt_buf1[0]; - salt_buf1_t[1] = salt_buf1[1]; - salt_buf1_t[2] = salt_buf1[2]; - salt_buf1_t[3] = salt_buf1[3]; - salt_buf2_t[0] = salt_buf2[0]; - salt_buf2_t[1] = salt_buf2[1]; - salt_buf2_t[2] = salt_buf2[2]; - salt_buf2_t[3] = salt_buf2[3]; - salt_buf3_t[0] = salt_buf3[0]; - salt_buf3_t[1] = salt_buf3[1]; - salt_buf3_t[2] = salt_buf3[2]; - salt_buf3_t[3] = salt_buf3[3]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_len); + md5_init (&ctx0); - w0[0] |= salt_buf0_t[0]; - w0[1] |= salt_buf0_t[1]; - w0[2] |= salt_buf0_t[2]; - w0[3] |= salt_buf0_t[3]; - w1[0] |= salt_buf1_t[0]; - w1[1] |= salt_buf1_t[1]; - w1[2] |= salt_buf1_t[2]; - w1[3] |= salt_buf1_t[3]; - w2[0] |= salt_buf2_t[0]; - w2[1] |= salt_buf2_t[1]; - w2[2] |= salt_buf2_t[2]; - w2[3] |= salt_buf2_t[3]; - w3[0] |= salt_buf3_t[0]; - w3[1] |= salt_buf3_t[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx1; - /** - * md5 - */ + md5_init_vector (&ctx1); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx1, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_update_vector (&ctx1, s, salt_len); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + md5_final_vector (&ctx1); - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + const u32x a = ctx1.h[0]; + const u32x b = ctx1.h[1]; + const u32x c = ctx1.h[2]; + const u32x d = ctx1.h[3]; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + md5_ctx_vector_t ctx; - /** - * prepend salt - */ + md5_init_vector_from_scalar (&ctx, &ctx0); - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - w3_t[2] = final_len * 8; - w3_t[3] = 0; + md5_final_vector (&ctx); - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m04110s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04110_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -367,6 +174,26 @@ void m04110s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -381,750 +208,113 @@ void m04110s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl }; /** - * salt + * base */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + const u32 pw_len = pws[gid].pw_len; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[ 0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[ 1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[ 2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[ 3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[ 4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[ 5]; - salt_buf1[2] = salt_bufs[salt_pos].salt_buf[ 6]; - salt_buf1[3] = salt_bufs[salt_pos].salt_buf[ 7]; - salt_buf2[0] = salt_bufs[salt_pos].salt_buf[ 8]; - salt_buf2[1] = salt_bufs[salt_pos].salt_buf[ 9]; - salt_buf2[2] = salt_bufs[salt_pos].salt_buf[10]; - salt_buf2[3] = salt_bufs[salt_pos].salt_buf[11]; - salt_buf3[0] = salt_bufs[salt_pos].salt_buf[12]; - salt_buf3[1] = salt_bufs[salt_pos].salt_buf[13]; - salt_buf3[2] = salt_bufs[salt_pos].salt_buf[14]; - salt_buf3[3] = salt_bufs[salt_pos].salt_buf[15]; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; - const u32 pw_salt_len = pw_len + salt_len; - const u32 final_len = salt_len + 32; + const u32 salt_lenv = ceil ((float) salt_len / 4); - /** - * append salt - */ + u32x s[64] = { 0 }; - u32 salt_buf0_t[4]; - u32 salt_buf1_t[4]; - u32 salt_buf2_t[4]; - u32 salt_buf3_t[4]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - salt_buf0_t[0] = salt_buf0[0]; - salt_buf0_t[1] = salt_buf0[1]; - salt_buf0_t[2] = salt_buf0[2]; - salt_buf0_t[3] = salt_buf0[3]; - salt_buf1_t[0] = salt_buf1[0]; - salt_buf1_t[1] = salt_buf1[1]; - salt_buf1_t[2] = salt_buf1[2]; - salt_buf1_t[3] = salt_buf1[3]; - salt_buf2_t[0] = salt_buf2[0]; - salt_buf2_t[1] = salt_buf2[1]; - salt_buf2_t[2] = salt_buf2[2]; - salt_buf2_t[3] = salt_buf2[3]; - salt_buf3_t[0] = salt_buf3[0]; - salt_buf3_t[1] = salt_buf3[1]; - salt_buf3_t[2] = salt_buf3[2]; - salt_buf3_t[3] = salt_buf3[3]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - append_0x80_4x4_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, salt_len); + md5_ctx_t ctx0; - switch_buffer_by_offset_le_S (salt_buf0_t, salt_buf1_t, salt_buf2_t, salt_buf3_t, pw_len); + md5_init (&ctx0); - w0[0] |= salt_buf0_t[0]; - w0[1] |= salt_buf0_t[1]; - w0[2] |= salt_buf0_t[2]; - w0[3] |= salt_buf0_t[3]; - w1[0] |= salt_buf1_t[0]; - w1[1] |= salt_buf1_t[1]; - w1[2] |= salt_buf1_t[2]; - w1[3] |= salt_buf1_t[3]; - w2[0] |= salt_buf2_t[0]; - w2[1] |= salt_buf2_t[1]; - w2[2] |= salt_buf2_t[2]; - w2[3] |= salt_buf2_t[3]; - w3[0] |= salt_buf3_t[0]; - w3[1] |= salt_buf3_t[1]; - w3[2] = pw_salt_len * 8; - w3[3] = 0; + md5_update_global (&ctx0, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; const u32x w0lr = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0lr; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx1; - /** - * md5 - */ + md5_init_vector (&ctx1); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx1, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_update_vector (&ctx1, s, salt_len); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + md5_final_vector (&ctx1); - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + const u32x a = ctx1.h[0]; + const u32x b = ctx1.h[1]; + const u32x c = ctx1.h[2]; + const u32x d = ctx1.h[3]; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + w0[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 + | uint_to_hex_lower8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 + | uint_to_hex_lower8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 + | uint_to_hex_lower8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 + | uint_to_hex_lower8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 + | uint_to_hex_lower8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 + | uint_to_hex_lower8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 + | uint_to_hex_lower8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 + | uint_to_hex_lower8 ((d >> 24) & 255) << 16; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; - w0_t[0] = uint_to_hex_lower8 ((a >> 0) & 255) << 0 - | uint_to_hex_lower8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_lower8 ((a >> 16) & 255) << 0 - | uint_to_hex_lower8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_lower8 ((b >> 0) & 255) << 0 - | uint_to_hex_lower8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_lower8 ((b >> 16) & 255) << 0 - | uint_to_hex_lower8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_lower8 ((c >> 0) & 255) << 0 - | uint_to_hex_lower8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_lower8 ((c >> 16) & 255) << 0 - | uint_to_hex_lower8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_lower8 ((d >> 0) & 255) << 0 - | uint_to_hex_lower8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_lower8 ((d >> 16) & 255) << 0 - | uint_to_hex_lower8 ((d >> 24) & 255) << 16; - w2_t[0] = 0x80; - w2_t[1] = 0; - w2_t[2] = 0; - w2_t[3] = 0; - w3_t[0] = 0; - w3_t[1] = 0; - w3_t[2] = 0; - w3_t[3] = 0; + md5_ctx_vector_t ctx; - /** - * prepend salt - */ + md5_init_vector_from_scalar (&ctx, &ctx0); - switch_buffer_by_offset_le (w0_t, w1_t, w2_t, w3_t, salt_len); + md5_update_vector_64 (&ctx, w0, w1, w2, w3, 32); - w3_t[2] = final_len * 8; - w3_t[3] = 0; + md5_final_vector (&ctx); - w0_t[0] |= salt_buf0[0]; - w0_t[1] |= salt_buf0[1]; - w0_t[2] |= salt_buf0[2]; - w0_t[3] |= salt_buf0[3]; - w1_t[0] |= salt_buf1[0]; - w1_t[1] |= salt_buf1[1]; - w1_t[2] |= salt_buf1[2]; - w1_t[3] |= salt_buf1[3]; - w2_t[0] |= salt_buf2[0]; - w2_t[1] |= salt_buf2[1]; - w2_t[2] |= salt_buf2[2]; - w2_t[3] |= salt_buf2[3]; - w3_t[0] |= salt_buf3[0]; - w3_t[1] |= salt_buf3[1]; - w3_t[2] |= salt_buf3[2]; - w3_t[3] |= salt_buf3[3]; + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - /** - * md5 - */ - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m04110_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04110_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04110_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04110m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04110_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04110_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04110_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'a' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'a' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m04110s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m04310_a0-optimized.cl b/OpenCL/m04310_a0-optimized.cl new file mode 100644 index 000000000..bd7706b61 --- /dev/null +++ b/OpenCL/m04310_a0-optimized.cl @@ -0,0 +1,606 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + w3[2] = out_len * 8; + w3[3] = 0; + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04310_a0-pure.cl b/OpenCL/m04310_a0-pure.cl deleted file mode 100644 index 8ff005fa0..000000000 --- a/OpenCL/m04310_a0-pure.cl +++ /dev/null @@ -1,272 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update (&ctx0, w, pw_len); - - md5_final (&ctx0); - - const u32 a = ctx0.h[0]; - const u32 b = ctx0.h[1]; - const u32 c = ctx0.h[2]; - const u32 d = ctx0.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04310_a0.cl b/OpenCL/m04310_a0.cl index bd7706b61..8ff005fa0 100644 --- a/OpenCL/m04310_a0.cl +++ b/OpenCL/m04310_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,7 +12,8 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) @@ -26,7 +27,7 @@ #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -59,256 +60,90 @@ __kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + u32 w[64] = { 0 }; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.len = 32; - w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + md5_update (&ctx, s, salt_len); - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; + md5_final (&ctx); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -337,52 +172,6 @@ __kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -395,212 +184,89 @@ __kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + md5_ctx_t ctx0; - append_0x80_2x4_VV (w0, w1, out_len); + md5_init (&ctx0); - w3[2] = out_len * 8; - w3[3] = 0; + md5_update (&ctx0, w, pw_len); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_final (&ctx0); - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + const u32 a = ctx0.h[0]; + const u32 b = ctx0.h[1]; + const u32 c = ctx0.h[2]; + const u32 d = ctx0.h[3]; - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + md5_ctx_t ctx; - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + md5_init (&ctx); - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.len = 32; - w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + md5_update (&ctx, s, salt_len); - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; + md5_final (&ctx); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04310_a1-optimized.cl b/OpenCL/m04310_a1-optimized.cl new file mode 100644 index 000000000..b5484387f --- /dev/null +++ b/OpenCL/m04310_a1-optimized.cl @@ -0,0 +1,722 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = pw_len * 8; + w3[3] = 0; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2[0] = salt_buf0[0]; + w2[1] = salt_buf0[1]; + w2[2] = salt_buf0[2]; + w2[3] = salt_buf0[3]; + w3[0] = salt_buf1[0]; + w3[1] = salt_buf1[1]; + w3[2] = (32 + salt_len) * 8; + w3[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + } +} + +__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m04310_a1-pure.cl b/OpenCL/m04310_a1-pure.cl deleted file mode 100644 index aea272f93..000000000 --- a/OpenCL/m04310_a1-pure.cl +++ /dev/null @@ -1,248 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - md5_ctx_t ctx0; - - md5_init (&ctx0); - - md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - md5_ctx_t ctx1 = ctx0; - - md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - md5_final (&ctx1); - - const u32 a = ctx1.h[0]; - const u32 b = ctx1.h[1]; - const u32 c = ctx1.h[2]; - const u32 d = ctx1.h[3]; - - md5_ctx_t ctx; - - md5_init (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update (&ctx, s, salt_len); - - md5_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04310_a1.cl b/OpenCL/m04310_a1.cl index b5484387f..aea272f93 100644 --- a/OpenCL/m04310_a1.cl +++ b/OpenCL/m04310_a1.cl @@ -3,14 +3,15 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -57,315 +58,79 @@ __kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + ctx.len = 32; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md5_update (&ctx, s, salt_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_final (&ctx); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -394,52 +159,6 @@ __kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; - - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * digest */ @@ -452,271 +171,78 @@ __kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + md5_ctx_t ctx0; + + md5_init (&ctx0); + + md5_update_global (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + md5_ctx_t ctx1 = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + md5_update_global (&ctx1, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + md5_final (&ctx1); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 a = ctx1.h[0]; + const u32 b = ctx1.h[1]; + const u32 c = ctx1.h[2]; + const u32 d = ctx1.h[3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; + md5_ctx_t ctx; - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; + md5_init (&ctx); - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } + ctx.len = 32; - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; + md5_update (&ctx, s, salt_len); - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = pw_len * 8; - w3[3] = 0; + md5_final (&ctx); - /** - * md5 - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; - - w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - w2[0] = salt_buf0[0]; - w2[1] = salt_buf0[1]; - w2[2] = salt_buf0[2]; - w2[3] = salt_buf0[3]; - w3[0] = salt_buf1[0]; - w3[1] = salt_buf1[1]; - w3[2] = (32 + salt_len) * 8; - w3[3] = 0; - - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; - - MD5_STEP (MD5_Fo, a, b, c, d, w0[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m04310_a3-optimized.cl b/OpenCL/m04310_a3-optimized.cl new file mode 100644 index 000000000..bdc09bb68 --- /dev/null +++ b/OpenCL/m04310_a3-optimized.cl @@ -0,0 +1,973 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#if VECT_SIZE == 1 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) +#elif VECT_SIZE == 2 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) +#elif VECT_SIZE == 4 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) +#elif VECT_SIZE == 8 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) +#elif VECT_SIZE == 16 +#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) +#endif + +void m04310m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = (32 + salt_len) * 8; + w3_t[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_M_SIMD (a, d, c, b); + } +} + +void m04310s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * salt + */ + + u32 salt_buf0[4]; + u32 salt_buf1[4]; + u32 salt_buf2[4]; + u32 salt_buf3[4]; + + salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; + salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; + salt_buf1[2] = 0; + salt_buf1[3] = 0; + salt_buf2[0] = 0; + salt_buf2[1] = 0; + salt_buf2[2] = 0; + salt_buf2[3] = 0; + salt_buf3[0] = 0; + salt_buf3[1] = 0; + salt_buf3[2] = 0; + salt_buf3[3] = 0; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0lr; + w0_t[1] = w0[1]; + w0_t[2] = w0[2]; + w0_t[3] = w0[3]; + w1_t[0] = w1[0]; + w1_t[1] = w1[1]; + w1_t[2] = w1[2]; + w1_t[3] = w1[3]; + w2_t[0] = w2[0]; + w2_t[1] = w2[1]; + w2_t[2] = w2[2]; + w2_t[3] = w2[3]; + w3_t[0] = w3[0]; + w3_t[1] = w3[1]; + w3_t[2] = w3[2]; + w3_t[3] = w3[3]; + + /** + * md5 + */ + + u32x a = MD5M_A; + u32x b = MD5M_B; + u32x c = MD5M_C; + u32x d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + a += MD5M_A; + b += MD5M_B; + c += MD5M_C; + d += MD5M_D; + + w0_t[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + w0_t[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + w0_t[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + w0_t[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + w1_t[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + w1_t[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + w1_t[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + w1_t[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + + w2_t[0] = salt_buf0[0]; + w2_t[1] = salt_buf0[1]; + w2_t[2] = salt_buf0[2]; + w2_t[3] = salt_buf0[3]; + w3_t[0] = salt_buf1[0]; + w3_t[1] = salt_buf1[1]; + w3_t[2] = (32 + salt_len) * 8; + w3_t[3] = 0; + + a = MD5M_A; + b = MD5M_B; + c = MD5M_C; + d = MD5M_D; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); + + if (MATCHES_NONE_VS (a, search[0])) continue; + + MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + + COMPARE_S_SIMD (a, d, c, b); + + } +} + +__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = pws[gid].i[14]; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} + +__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * modifier + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = pws[gid].i[14]; + w3[3] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * bin2asc table + */ + + __local u32 l_bin2asc[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * main + */ + + m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); +} diff --git a/OpenCL/m04310_a3-pure.cl b/OpenCL/m04310_a3-pure.cl deleted file mode 100644 index 2f491a2fb..000000000 --- a/OpenCL/m04310_a3-pure.cl +++ /dev/null @@ -1,282 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_md5.cl" - -#if VECT_SIZE == 1 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) -#elif VECT_SIZE == 2 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1]) -#elif VECT_SIZE == 4 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3]) -#elif VECT_SIZE == 8 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7]) -#elif VECT_SIZE == 16 -#define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) -#endif - -__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32x s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - md5_ctx_vector_t ctx0; - - md5_init_vector (&ctx0); - - md5_update_vector (&ctx0, w, pw_len); - - md5_final_vector (&ctx0); - - const u32x a = ctx0.h[0]; - const u32x b = ctx0.h[1]; - const u32x c = ctx0.h[2]; - const u32x d = ctx0.h[3]; - - md5_ctx_vector_t ctx; - - md5_init_vector (&ctx); - - ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - - ctx.len = 32; - - md5_update_vector (&ctx, s, salt_len); - - md5_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m04310_a3.cl b/OpenCL/m04310_a3.cl index bdc09bb68..2f491a2fb 100644 --- a/OpenCL/m04310_a3.cl +++ b/OpenCL/m04310_a3.cl @@ -11,6 +11,7 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_md5.cl" #if VECT_SIZE == 1 #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i)]) @@ -24,7 +25,7 @@ #define uint_to_hex_upper8(i) (u32x) (l_bin2asc[(i).s0], l_bin2asc[(i).s1], l_bin2asc[(i).s2], l_bin2asc[(i).s3], l_bin2asc[(i).s4], l_bin2asc[(i).s5], l_bin2asc[(i).s6], l_bin2asc[(i).s7], l_bin2asc[(i).s8], l_bin2asc[(i).s9], l_bin2asc[(i).sa], l_bin2asc[(i).sb], l_bin2asc[(i).sc], l_bin2asc[(i).sd], l_bin2asc[(i).se], l_bin2asc[(i).sf]) #endif -void m04310m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04310_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -32,255 +33,121 @@ void m04310m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; + + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - w0_t[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + ctx.len = 32; - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = (32 + salt_len) * 8; - w3_t[3] = 0; + md5_update_vector (&ctx, s, salt_len); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_final_vector (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_M_SIMD (a, d, c, b); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m04310s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 *l_bin2asc) +__kernel void m04310_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -288,34 +155,26 @@ void m04310s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); /** - * salt + * bin2asc table */ - u32 salt_buf0[4]; - u32 salt_buf1[4]; - u32 salt_buf2[4]; - u32 salt_buf3[4]; + __local u32 l_bin2asc[256]; - salt_buf0[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf0[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf0[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf0[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf1[0] = salt_bufs[salt_pos].salt_buf[4]; - salt_buf1[1] = salt_bufs[salt_pos].salt_buf[5]; - salt_buf1[2] = 0; - salt_buf1[3] = 0; - salt_buf2[0] = 0; - salt_buf2[1] = 0; - salt_buf2[2] = 0; - salt_buf2[3] = 0; - salt_buf3[0] = 0; - salt_buf3[1] = 0; - salt_buf3[2] = 0; - salt_buf3[3] = 0; + for (u32 i = lid; i < 256; i += lsz) + { + const u32 i0 = (i >> 0) & 15; + const u32 i1 = (i >> 4) & 15; - const u32 salt_len = salt_bufs[salt_pos].salt_len; + l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 + | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -329,645 +188,95 @@ void m04310s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32x s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0lr; - w0_t[1] = w0[1]; - w0_t[2] = w0[2]; - w0_t[3] = w0[3]; - w1_t[0] = w1[0]; - w1_t[1] = w1[1]; - w1_t[2] = w1[2]; - w1_t[3] = w1[3]; - w2_t[0] = w2[0]; - w2_t[1] = w2[1]; - w2_t[2] = w2[2]; - w2_t[3] = w2[3]; - w3_t[0] = w3[0]; - w3_t[1] = w3[1]; - w3_t[2] = w3[2]; - w3_t[3] = w3[3]; + md5_ctx_vector_t ctx0; - /** - * md5 - */ + md5_init_vector (&ctx0); - u32x a = MD5M_A; - u32x b = MD5M_B; - u32x c = MD5M_C; - u32x d = MD5M_D; + md5_update_vector (&ctx0, w, pw_len); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); + md5_final_vector (&ctx0); - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); + const u32x a = ctx0.h[0]; + const u32x b = ctx0.h[1]; + const u32x c = ctx0.h[2]; + const u32x d = ctx0.h[3]; - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); + md5_ctx_vector_t ctx; - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); + md5_init_vector (&ctx); - a += MD5M_A; - b += MD5M_B; - c += MD5M_C; - d += MD5M_D; + ctx.w0[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 + | uint_to_hex_upper8 ((a >> 8) & 255) << 16; + ctx.w0[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 + | uint_to_hex_upper8 ((a >> 24) & 255) << 16; + ctx.w0[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 + | uint_to_hex_upper8 ((b >> 8) & 255) << 16; + ctx.w0[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 + | uint_to_hex_upper8 ((b >> 24) & 255) << 16; + ctx.w1[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 + | uint_to_hex_upper8 ((c >> 8) & 255) << 16; + ctx.w1[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 + | uint_to_hex_upper8 ((c >> 24) & 255) << 16; + ctx.w1[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 + | uint_to_hex_upper8 ((d >> 8) & 255) << 16; + ctx.w1[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 + | uint_to_hex_upper8 ((d >> 24) & 255) << 16; - w0_t[0] = uint_to_hex_upper8 ((a >> 0) & 255) << 0 - | uint_to_hex_upper8 ((a >> 8) & 255) << 16; - w0_t[1] = uint_to_hex_upper8 ((a >> 16) & 255) << 0 - | uint_to_hex_upper8 ((a >> 24) & 255) << 16; - w0_t[2] = uint_to_hex_upper8 ((b >> 0) & 255) << 0 - | uint_to_hex_upper8 ((b >> 8) & 255) << 16; - w0_t[3] = uint_to_hex_upper8 ((b >> 16) & 255) << 0 - | uint_to_hex_upper8 ((b >> 24) & 255) << 16; - w1_t[0] = uint_to_hex_upper8 ((c >> 0) & 255) << 0 - | uint_to_hex_upper8 ((c >> 8) & 255) << 16; - w1_t[1] = uint_to_hex_upper8 ((c >> 16) & 255) << 0 - | uint_to_hex_upper8 ((c >> 24) & 255) << 16; - w1_t[2] = uint_to_hex_upper8 ((d >> 0) & 255) << 0 - | uint_to_hex_upper8 ((d >> 8) & 255) << 16; - w1_t[3] = uint_to_hex_upper8 ((d >> 16) & 255) << 0 - | uint_to_hex_upper8 ((d >> 24) & 255) << 16; + ctx.len = 32; - w2_t[0] = salt_buf0[0]; - w2_t[1] = salt_buf0[1]; - w2_t[2] = salt_buf0[2]; - w2_t[3] = salt_buf0[3]; - w3_t[0] = salt_buf1[0]; - w3_t[1] = salt_buf1[1]; - w3_t[2] = (32 + salt_len) * 8; - w3_t[3] = 0; + md5_update_vector (&ctx, s, salt_len); - a = MD5M_A; - b = MD5M_B; - c = MD5M_C; - d = MD5M_D; + md5_final_vector (&ctx); - MD5_STEP (MD5_Fo, a, b, c, d, w0_t[0], MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w0_t[1], MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w0_t[2], MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w0_t[3], MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w1_t[0], MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t[1], MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w1_t[2], MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w1_t[3], MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w2_t[0], MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w2_t[1], MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t[2], MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w2_t[3], MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w3_t[0], MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w3_t[1], MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w3_t[2], MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t[3], MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w0_t[1], MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w1_t[2], MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w2_t[3], MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t[0], MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w1_t[1], MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t[2], MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t[3], MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w1_t[0], MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w2_t[1], MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w3_t[2], MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w0_t[3], MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w2_t[0], MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w3_t[1], MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w0_t[2], MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w1_t[3], MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w3_t[0], MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w1_t[1], MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w2_t[0], MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w2_t[3], MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w3_t[2], MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w0_t[1], MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w1_t[0], MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w1_t[3], MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t[2], MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w3_t[1], MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t[0], MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w0_t[3], MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w1_t[2], MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w2_t[1], MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w3_t[0], MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t[3], MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w0_t[2], MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t[0], MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w1_t[3], MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w3_t[2], MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t[1], MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w3_t[0], MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w0_t[3], MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t[2], MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w0_t[1], MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w2_t[0], MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t[3], MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w1_t[2], MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w3_t[1], MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w1_t[0], MD5C3c, MD5S30); - - if (MATCHES_NONE_VS (a, search[0])) continue; - - MD5_STEP (MD5_I , d, a, b, c, w2_t[3], MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w0_t[2], MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w2_t[1], MD5C3f, MD5S33); - - COMPARE_S_SIMD (a, d, c, b); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m04310_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04310_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04310_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04310_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04310_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = pws[gid].i[14]; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} - -__kernel void m04310_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * modifier - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = pws[gid].i[14]; - w3[3] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * bin2asc table - */ - - __local u32 l_bin2asc[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - const u32 i0 = (i >> 0) & 15; - const u32 i1 = (i >> 4) & 15; - - l_bin2asc[i] = ((i0 < 10) ? '0' + i0 : 'A' - 10 + i0) << 8 - | ((i1 < 10) ? '0' + i1 : 'A' - 10 + i1) << 0; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * main - */ - - m04310s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, l_bin2asc); -} diff --git a/OpenCL/m05800-pure.cl b/OpenCL/m05800-optimized.cl similarity index 91% rename from OpenCL/m05800-pure.cl rename to OpenCL/m05800-optimized.cl index 3e9a9c72d..b84015eef 100644 --- a/OpenCL/m05800-pure.cl +++ b/OpenCL/m05800-optimized.cl @@ -8,7 +8,6 @@ #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_hash_sha1.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" @@ -2194,7 +2193,7 @@ void append_salt (u32 w0[4], u32 w1[4], u32 w2[4], const u32 append[5], const u3 } } -void orig_sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) +void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { u32 A = digest[0]; u32 B = digest[1]; @@ -2324,32 +2323,91 @@ void orig_sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], con __kernel void m05800_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global androidpin_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { + /** + * base + */ + const u32 gid = get_global_id (0); if (gid >= gid_max) return; - sha1_ctx_t ctx; + u32 word_buf[4]; - sha1_init (&ctx); + word_buf[0] = pws[gid].i[ 0]; + word_buf[1] = pws[gid].i[ 1]; + word_buf[2] = pws[gid].i[ 2]; + word_buf[3] = pws[gid].i[ 3]; - u32 w0[4] = { 0 }; w0[0] = 0x30000000; - u32 w1[4] = { 0 }; - u32 w2[4] = { 0 }; - u32 w3[4] = { 0 }; + const u32 pw_len = pws[gid].pw_len; - sha1_update_64 (&ctx, w0, w1, w2, w3, 1); + /** + * salt + */ - sha1_update_global_swap (&ctx, pws[gid].i, pws[gid].pw_len); + u32 salt_len = salt_bufs[salt_pos].salt_len; - sha1_update_global_swap (&ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + u32 salt_buf[5]; - sha1_final (&ctx); + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf[4] = salt_bufs[salt_pos].salt_buf[4]; - tmps[gid].digest_buf[0] = ctx.h[0]; - tmps[gid].digest_buf[1] = ctx.h[1]; - tmps[gid].digest_buf[2] = ctx.h[2]; - tmps[gid].digest_buf[3] = ctx.h[3]; - tmps[gid].digest_buf[4] = ctx.h[4]; + /** + * init + */ + + const u32 pc_len = 1; + const u32 pc_dec = 0x30; + + u32 data0[4] = { 0, 0, 0, 0 }; + u32 data1[4] = { 0, 0, 0, 0 }; + u32 data2[4] = { 0, 0, 0, 0 }; + + data0[0] = pc_dec; + + append_word (data0, data1, word_buf, pc_len); + + append_salt (data0, data1, data2, salt_buf, pc_len + pw_len); + + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; + + w0[0] = swap32 (data0[0]); + w0[1] = swap32 (data0[1]); + w0[2] = swap32 (data0[2]); + w0[3] = swap32 (data0[3]); + w1[0] = swap32 (data1[0]); + w1[1] = swap32 (data1[1]); + w1[2] = swap32 (data1[2]); + w1[3] = swap32 (data1[3]); + w2[0] = swap32 (data2[0]); + w2[1] = swap32 (data2[1]); + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = (pc_len + pw_len + salt_len) * 8; + + u32 digest[5]; + + digest[0] = SHA1M_A; + digest[1] = SHA1M_B; + digest[2] = SHA1M_C; + digest[3] = SHA1M_D; + digest[4] = SHA1M_E; + + sha1_transform (w0, w1, w2, w3, digest); + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; + tmps[gid].digest_buf[4] = digest[4]; } __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global androidpin_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -2380,35 +2438,18 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; /** - * init + * base */ + u32 word_buf[4]; + + word_buf[0] = pws[gid].i[ 0]; + word_buf[1] = pws[gid].i[ 1]; + word_buf[2] = pws[gid].i[ 2]; + word_buf[3] = pws[gid].i[ 3]; + const u32 pw_len = pws[gid].pw_len; - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32 (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32 (salt_bufs[salt_pos].salt_buf[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - u32 digest[5]; digest[0] = tmps[gid].digest_buf[0]; @@ -2417,6 +2458,20 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul digest[3] = tmps[gid].digest_buf[3]; digest[4] = tmps[gid].digest_buf[4]; + /** + * salt + */ + + u32 salt_len = salt_bufs[salt_pos].salt_len; + + u32 salt_buf[5]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; + salt_buf[4] = salt_bufs[salt_pos].salt_buf[4]; + /** * loop */ @@ -2426,30 +2481,45 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul const u32 pc_dec = s_pc_dec[j]; const u32 pc_len = s_pc_len[j]; - sha1_ctx_t ctx; + u32 data0[4] = { 0, 0, 0, 0 }; + u32 data1[4] = { 0, 0, 0, 0 }; + u32 data2[4] = { 0, 0, 0, 0 }; - sha1_init (&ctx); + data0[0] = pc_dec; - ctx.w0[0] = digest[0]; - ctx.w0[1] = digest[1]; - ctx.w0[2] = digest[2]; - ctx.w0[3] = digest[3]; - ctx.w1[0] = digest[4]; - ctx.w1[1] = swap32 (pc_dec); + append_word (data0, data1, word_buf, pc_len); - ctx.len = 20 + pc_len; + append_salt (data0, data1, data2, salt_buf, pc_len + pw_len); - sha1_update (&ctx, w, pw_len); + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; - sha1_update (&ctx, s, salt_len); + w0[0] = digest[0]; + w0[1] = digest[1]; + w0[2] = digest[2]; + w0[3] = digest[3]; + w1[0] = digest[4]; + w1[1] = swap32 (data0[0]); + w1[2] = swap32 (data0[1]); + w1[3] = swap32 (data0[2]); + w2[0] = swap32 (data0[3]); + w2[1] = swap32 (data1[0]); + w2[2] = swap32 (data1[1]); + w2[3] = swap32 (data1[2]); + w3[0] = swap32 (data1[3]); + w3[1] = swap32 (data2[0]); + w3[2] = 0; + w3[3] = (20 + pc_len + pw_len + salt_len) * 8; - sha1_final (&ctx); + digest[0] = SHA1M_A; + digest[1] = SHA1M_B; + digest[2] = SHA1M_C; + digest[3] = SHA1M_D; + digest[4] = SHA1M_E; - digest[0] = ctx.h[0]; - digest[1] = ctx.h[1]; - digest[2] = ctx.h[2]; - digest[3] = ctx.h[3]; - digest[4] = ctx.h[4]; + sha1_transform (w0, w1, w2, w3, digest); } tmps[gid].digest_buf[0] = digest[0]; diff --git a/OpenCL/m05800.cl b/OpenCL/m05800.cl index b84015eef..3e9a9c72d 100644 --- a/OpenCL/m05800.cl +++ b/OpenCL/m05800.cl @@ -8,6 +8,7 @@ #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_sha1.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" @@ -2193,7 +2194,7 @@ void append_salt (u32 w0[4], u32 w1[4], u32 w2[4], const u32 append[5], const u3 } } -void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) +void orig_sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[5]) { u32 A = digest[0]; u32 B = digest[1]; @@ -2323,91 +2324,32 @@ void sha1_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u3 __kernel void m05800_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global androidpin_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 word_buf[4]; + sha1_ctx_t ctx; - word_buf[0] = pws[gid].i[ 0]; - word_buf[1] = pws[gid].i[ 1]; - word_buf[2] = pws[gid].i[ 2]; - word_buf[3] = pws[gid].i[ 3]; + sha1_init (&ctx); - const u32 pw_len = pws[gid].pw_len; + u32 w0[4] = { 0 }; w0[0] = 0x30000000; + u32 w1[4] = { 0 }; + u32 w2[4] = { 0 }; + u32 w3[4] = { 0 }; - /** - * salt - */ + sha1_update_64 (&ctx, w0, w1, w2, w3, 1); - u32 salt_len = salt_bufs[salt_pos].salt_len; + sha1_update_global_swap (&ctx, pws[gid].i, pws[gid].pw_len); - u32 salt_buf[5]; + sha1_update_global_swap (&ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf[4] = salt_bufs[salt_pos].salt_buf[4]; + sha1_final (&ctx); - /** - * init - */ - - const u32 pc_len = 1; - const u32 pc_dec = 0x30; - - u32 data0[4] = { 0, 0, 0, 0 }; - u32 data1[4] = { 0, 0, 0, 0 }; - u32 data2[4] = { 0, 0, 0, 0 }; - - data0[0] = pc_dec; - - append_word (data0, data1, word_buf, pc_len); - - append_salt (data0, data1, data2, salt_buf, pc_len + pw_len); - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - w0[0] = swap32 (data0[0]); - w0[1] = swap32 (data0[1]); - w0[2] = swap32 (data0[2]); - w0[3] = swap32 (data0[3]); - w1[0] = swap32 (data1[0]); - w1[1] = swap32 (data1[1]); - w1[2] = swap32 (data1[2]); - w1[3] = swap32 (data1[3]); - w2[0] = swap32 (data2[0]); - w2[1] = swap32 (data2[1]); - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = (pc_len + pw_len + salt_len) * 8; - - u32 digest[5]; - - digest[0] = SHA1M_A; - digest[1] = SHA1M_B; - digest[2] = SHA1M_C; - digest[3] = SHA1M_D; - digest[4] = SHA1M_E; - - sha1_transform (w0, w1, w2, w3, digest); - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; - tmps[gid].digest_buf[4] = digest[4]; + tmps[gid].digest_buf[0] = ctx.h[0]; + tmps[gid].digest_buf[1] = ctx.h[1]; + tmps[gid].digest_buf[2] = ctx.h[2]; + tmps[gid].digest_buf[3] = ctx.h[3]; + tmps[gid].digest_buf[4] = ctx.h[4]; } __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global androidpin_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -2438,18 +2380,35 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; /** - * base + * init */ - u32 word_buf[4]; - - word_buf[0] = pws[gid].i[ 0]; - word_buf[1] = pws[gid].i[ 1]; - word_buf[2] = pws[gid].i[ 2]; - word_buf[3] = pws[gid].i[ 3]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32 (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32 (salt_bufs[salt_pos].salt_buf[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + u32 digest[5]; digest[0] = tmps[gid].digest_buf[0]; @@ -2458,20 +2417,6 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul digest[3] = tmps[gid].digest_buf[3]; digest[4] = tmps[gid].digest_buf[4]; - /** - * salt - */ - - u32 salt_len = salt_bufs[salt_pos].salt_len; - - u32 salt_buf[5]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; - salt_buf[4] = salt_bufs[salt_pos].salt_buf[4]; - /** * loop */ @@ -2481,45 +2426,30 @@ __kernel void m05800_loop (__global pw_t *pws, __global const kernel_rule_t *rul const u32 pc_dec = s_pc_dec[j]; const u32 pc_len = s_pc_len[j]; - u32 data0[4] = { 0, 0, 0, 0 }; - u32 data1[4] = { 0, 0, 0, 0 }; - u32 data2[4] = { 0, 0, 0, 0 }; + sha1_ctx_t ctx; - data0[0] = pc_dec; + sha1_init (&ctx); - append_word (data0, data1, word_buf, pc_len); + ctx.w0[0] = digest[0]; + ctx.w0[1] = digest[1]; + ctx.w0[2] = digest[2]; + ctx.w0[3] = digest[3]; + ctx.w1[0] = digest[4]; + ctx.w1[1] = swap32 (pc_dec); - append_salt (data0, data1, data2, salt_buf, pc_len + pw_len); + ctx.len = 20 + pc_len; - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; + sha1_update (&ctx, w, pw_len); - w0[0] = digest[0]; - w0[1] = digest[1]; - w0[2] = digest[2]; - w0[3] = digest[3]; - w1[0] = digest[4]; - w1[1] = swap32 (data0[0]); - w1[2] = swap32 (data0[1]); - w1[3] = swap32 (data0[2]); - w2[0] = swap32 (data0[3]); - w2[1] = swap32 (data1[0]); - w2[2] = swap32 (data1[1]); - w2[3] = swap32 (data1[2]); - w3[0] = swap32 (data1[3]); - w3[1] = swap32 (data2[0]); - w3[2] = 0; - w3[3] = (20 + pc_len + pw_len + salt_len) * 8; + sha1_update (&ctx, s, salt_len); - digest[0] = SHA1M_A; - digest[1] = SHA1M_B; - digest[2] = SHA1M_C; - digest[3] = SHA1M_D; - digest[4] = SHA1M_E; + sha1_final (&ctx); - sha1_transform (w0, w1, w2, w3, digest); + digest[0] = ctx.h[0]; + digest[1] = ctx.h[1]; + digest[2] = ctx.h[2]; + digest[3] = ctx.h[3]; + digest[4] = ctx.h[4]; } tmps[gid].digest_buf[0] = digest[0]; diff --git a/OpenCL/m06100_a0-optimized.cl b/OpenCL/m06100_a0-optimized.cl new file mode 100644 index 000000000..c1e722e27 --- /dev/null +++ b/OpenCL/m06100_a0-optimized.cl @@ -0,0 +1,1618 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +#define R 10 + +#if VECT_SIZE == 1 +#define BOX(S,n,i) (S)[(n)][(i)] +#elif VECT_SIZE == 2 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) +#elif VECT_SIZE == 4 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) +#elif VECT_SIZE == 8 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) +#elif VECT_SIZE == 16 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) +#endif + +__constant u32a Ch[8][256] = +{ + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + } +}; + +__constant u32a Cl[8][256] = +{ + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + }, + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, +}; + +__constant u32a rch[R + 1] = +{ + 0x00000000, + 0x1823c6e8, + 0x36a6d2f5, + 0x60bc9b8e, + 0x1de0d7c2, + 0x157737e5, + 0x58c9290a, + 0xbd5d10f4, + 0xe427418b, + 0xfbee7c66, + 0xca2dbf07, +}; + +__constant u32a rcl[R + 1] = +{ + 0x00000000, + 0x87b8014f, + 0x796f9152, + 0xa30c7b35, + 0x2e4bfe57, + 0x9ff04ada, + 0xb1a06b85, + 0xcb3e0567, + 0xa77d95d8, + 0xdd17479e, + 0xad5a8333, +}; + +// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed + +void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) +{ + u32x Kh[8]; + u32x Kl[8]; + + Kh[0] = 0x300beec0; + Kl[0] = 0xaf902967; + Kh[1] = 0x28282828; + Kl[1] = 0x28282828; + Kh[2] = 0x28282828; + Kl[2] = 0x28282828; + Kh[3] = 0x28282828; + Kl[3] = 0x28282828; + Kh[4] = 0x28282828; + Kl[4] = 0x28282828; + Kh[5] = 0x28282828; + Kl[5] = 0x28282828; + Kh[6] = 0x28282828; + Kl[6] = 0x28282828; + Kh[7] = 0x28282828; + Kl[7] = 0x28282828; + + u32x stateh[8]; + u32x statel[8]; + + stateh[0] = w[ 0]; + statel[0] = w[ 1]; + stateh[1] = w[ 2]; + statel[1] = w[ 3]; + stateh[2] = w[ 4]; + statel[2] = w[ 5]; + stateh[3] = w[ 6]; + statel[3] = w[ 7]; + stateh[4] = w[ 8]; + statel[4] = w[ 9]; + stateh[5] = w[10]; + statel[5] = w[11]; + stateh[6] = w[12]; + statel[6] = w[13]; + stateh[7] = w[14]; + statel[7] = w[15]; + + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + + for (int r = 2; r <= R; r++) + { + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = Kh[(i + 8) & 7] >> 24; + const u32x Lp1 = Kh[(i + 7) & 7] >> 16; + const u32x Lp2 = Kh[(i + 6) & 7] >> 8; + const u32x Lp3 = Kh[(i + 5) & 7] >> 0; + const u32x Lp4 = Kl[(i + 4) & 7] >> 24; + const u32x Lp5 = Kl[(i + 3) & 7] >> 16; + const u32x Lp6 = Kl[(i + 2) & 7] >> 8; + const u32x Lp7 = Kl[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + Kh[0] = Lh[0] ^ rch[r]; + Kl[0] = Ll[0] ^ rcl[r]; + Kh[1] = Lh[1]; + Kl[1] = Ll[1]; + Kh[2] = Lh[2]; + Kl[2] = Ll[2]; + Kh[3] = Lh[3]; + Kl[3] = Ll[3]; + Kh[4] = Lh[4]; + Kl[4] = Ll[4]; + Kh[5] = Lh[5]; + Kl[5] = Ll[5]; + Kh[6] = Lh[6]; + Kl[6] = Ll[6]; + Kh[7] = Lh[7]; + Kl[7] = Ll[7]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + } + + dgst[ 0] = stateh[0] ^ w[ 0]; + dgst[ 1] = statel[0] ^ w[ 1]; + dgst[ 2] = stateh[1] ^ w[ 2]; + dgst[ 3] = statel[1] ^ w[ 3]; + dgst[ 4] = stateh[2] ^ w[ 4]; + dgst[ 5] = statel[2] ^ w[ 5]; + dgst[ 6] = stateh[3] ^ w[ 6]; + dgst[ 7] = statel[3] ^ w[ 7]; + dgst[ 8] = stateh[4] ^ w[ 8]; + dgst[ 9] = statel[4] ^ w[ 9]; + dgst[10] = stateh[5] ^ w[10]; + dgst[11] = statel[5] ^ w[11]; + dgst[12] = stateh[6] ^ w[12]; + dgst[13] = statel[6] ^ w[13]; + dgst[14] = stateh[7] ^ w[14]; + dgst[15] = statel[7] ^ w[15]; +} + +__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + u32x w[16]; + + w[ 0] = swap32 (w0[0]); + w[ 1] = swap32 (w0[1]); + w[ 2] = swap32 (w0[2]); + w[ 3] = swap32 (w0[3]); + w[ 4] = swap32 (w1[0]); + w[ 5] = swap32 (w1[1]); + w[ 6] = swap32 (w1[2]); + w[ 7] = swap32 (w1[3]); + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = out_len * 8; + + /** + * Whirlpool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + u32x w[16]; + + w[ 0] = swap32 (w0[0]); + w[ 1] = swap32 (w0[1]); + w[ 2] = swap32 (w0[2]); + w[ 3] = swap32 (w0[3]); + w[ 4] = swap32 (w1[0]); + w[ 5] = swap32 (w1[1]); + w[ 6] = swap32 (w1[2]); + w[ 7] = swap32 (w1[3]); + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = out_len * 8; + + /** + * Whirlpool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m06100_a0-pure.cl b/OpenCL/m06100_a0-pure.cl deleted file mode 100644 index 48fcad611..000000000 --- a/OpenCL/m06100_a0-pure.cl +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_whirlpool.cl" - -__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - whirlpool_ctx_t ctx; - - whirlpool_init (&ctx, s_Ch, s_Cl); - - whirlpool_update (&ctx, w, pw_len); - - whirlpool_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - whirlpool_ctx_t ctx; - - whirlpool_init (&ctx, s_Ch, s_Cl); - - whirlpool_update (&ctx, w, pw_len); - - whirlpool_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m06100_a0.cl b/OpenCL/m06100_a0.cl index c1e722e27..48fcad611 100644 --- a/OpenCL/m06100_a0.cl +++ b/OpenCL/m06100_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,1338 +12,10 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_whirlpool.cl" -#define R 10 - -#if VECT_SIZE == 1 -#define BOX(S,n,i) (S)[(n)][(i)] -#elif VECT_SIZE == 2 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) -#elif VECT_SIZE == 4 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) -#elif VECT_SIZE == 8 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) -#elif VECT_SIZE == 16 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) -#endif - -__constant u32a Ch[8][256] = -{ - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - } -}; - -__constant u32a Cl[8][256] = -{ - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - }, - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, -}; - -__constant u32a rch[R + 1] = -{ - 0x00000000, - 0x1823c6e8, - 0x36a6d2f5, - 0x60bc9b8e, - 0x1de0d7c2, - 0x157737e5, - 0x58c9290a, - 0xbd5d10f4, - 0xe427418b, - 0xfbee7c66, - 0xca2dbf07, -}; - -__constant u32a rcl[R + 1] = -{ - 0x00000000, - 0x87b8014f, - 0x796f9152, - 0xa30c7b35, - 0x2e4bfe57, - 0x9ff04ada, - 0xb1a06b85, - 0xcb3e0567, - 0xa77d95d8, - 0xdd17479e, - 0xad5a8333, -}; - -// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed - -void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) -{ - u32x Kh[8]; - u32x Kl[8]; - - Kh[0] = 0x300beec0; - Kl[0] = 0xaf902967; - Kh[1] = 0x28282828; - Kl[1] = 0x28282828; - Kh[2] = 0x28282828; - Kl[2] = 0x28282828; - Kh[3] = 0x28282828; - Kl[3] = 0x28282828; - Kh[4] = 0x28282828; - Kl[4] = 0x28282828; - Kh[5] = 0x28282828; - Kl[5] = 0x28282828; - Kh[6] = 0x28282828; - Kl[6] = 0x28282828; - Kh[7] = 0x28282828; - Kl[7] = 0x28282828; - - u32x stateh[8]; - u32x statel[8]; - - stateh[0] = w[ 0]; - statel[0] = w[ 1]; - stateh[1] = w[ 2]; - statel[1] = w[ 3]; - stateh[2] = w[ 4]; - statel[2] = w[ 5]; - stateh[3] = w[ 6]; - statel[3] = w[ 7]; - stateh[4] = w[ 8]; - statel[4] = w[ 9]; - stateh[5] = w[10]; - statel[5] = w[11]; - stateh[6] = w[12]; - statel[6] = w[13]; - stateh[7] = w[14]; - statel[7] = w[15]; - - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - - for (int r = 2; r <= R; r++) - { - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = Kh[(i + 8) & 7] >> 24; - const u32x Lp1 = Kh[(i + 7) & 7] >> 16; - const u32x Lp2 = Kh[(i + 6) & 7] >> 8; - const u32x Lp3 = Kh[(i + 5) & 7] >> 0; - const u32x Lp4 = Kl[(i + 4) & 7] >> 24; - const u32x Lp5 = Kl[(i + 3) & 7] >> 16; - const u32x Lp6 = Kl[(i + 2) & 7] >> 8; - const u32x Lp7 = Kl[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - Kh[0] = Lh[0] ^ rch[r]; - Kl[0] = Ll[0] ^ rcl[r]; - Kh[1] = Lh[1]; - Kl[1] = Ll[1]; - Kh[2] = Lh[2]; - Kl[2] = Ll[2]; - Kh[3] = Lh[3]; - Kl[3] = Ll[3]; - Kh[4] = Lh[4]; - Kl[4] = Ll[4]; - Kh[5] = Lh[5]; - Kl[5] = Ll[5]; - Kh[6] = Lh[6]; - Kl[6] = Ll[6]; - Kh[7] = Lh[7]; - Kl[7] = Ll[7]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - } - - dgst[ 0] = stateh[0] ^ w[ 0]; - dgst[ 1] = statel[0] ^ w[ 1]; - dgst[ 2] = stateh[1] ^ w[ 2]; - dgst[ 3] = statel[1] ^ w[ 3]; - dgst[ 4] = stateh[2] ^ w[ 4]; - dgst[ 5] = statel[2] ^ w[ 5]; - dgst[ 6] = stateh[3] ^ w[ 6]; - dgst[ 7] = statel[3] ^ w[ 7]; - dgst[ 8] = stateh[4] ^ w[ 8]; - dgst[ 9] = statel[4] ^ w[ 9]; - dgst[10] = stateh[5] ^ w[10]; - dgst[11] = statel[5] ^ w[11]; - dgst[12] = stateh[6] ^ w[12]; - dgst[13] = statel[6] ^ w[13]; - dgst[14] = stateh[7] ^ w[14]; - dgst[15] = statel[7] ^ w[15]; -} - -__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1389,92 +61,45 @@ __kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + whirlpool_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + whirlpool_init (&ctx, s_Ch, s_Cl); - u32x w[16]; + whirlpool_update (&ctx, w, pw_len); - w[ 0] = swap32 (w0[0]); - w[ 1] = swap32 (w0[1]); - w[ 2] = swap32 (w0[2]); - w[ 3] = swap32 (w0[3]); - w[ 4] = swap32 (w1[0]); - w[ 5] = swap32 (w1[1]); - w[ 6] = swap32 (w1[2]); - w[ 7] = swap32 (w1[3]); - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = out_len * 8; + whirlpool_final (&ctx); - /** - * Whirlpool - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x dgst[16]; - - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; - - whirlpool_transform (w, dgst, s_Ch, s_Cl); - - COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1516,24 +141,6 @@ __kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -1546,73 +153,44 @@ __kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + whirlpool_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + whirlpool_init (&ctx, s_Ch, s_Cl); - u32x w[16]; + whirlpool_update (&ctx, w, pw_len); - w[ 0] = swap32 (w0[0]); - w[ 1] = swap32 (w0[1]); - w[ 2] = swap32 (w0[2]); - w[ 3] = swap32 (w0[3]); - w[ 4] = swap32 (w1[0]); - w[ 5] = swap32 (w1[1]); - w[ 6] = swap32 (w1[2]); - w[ 7] = swap32 (w1[3]); - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = out_len * 8; + whirlpool_final (&ctx); - /** - * Whirlpool - */ + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - u32x dgst[16]; - - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; - - whirlpool_transform (w, dgst, s_Ch, s_Cl); - - COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m06100_a1-optimized.cl b/OpenCL/m06100_a1-optimized.cl new file mode 100644 index 000000000..9732930fd --- /dev/null +++ b/OpenCL/m06100_a1-optimized.cl @@ -0,0 +1,1732 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define R 10 + +#if VECT_SIZE == 1 +#define BOX(S,n,i) (S)[(n)][(i)] +#elif VECT_SIZE == 2 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) +#elif VECT_SIZE == 4 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) +#elif VECT_SIZE == 8 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) +#elif VECT_SIZE == 16 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) +#endif + +__constant u32a Ch[8][256] = +{ + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + } +}; + +__constant u32a Cl[8][256] = +{ + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + }, + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, +}; + +__constant u32a rch[R + 1] = +{ + 0x00000000, + 0x1823c6e8, + 0x36a6d2f5, + 0x60bc9b8e, + 0x1de0d7c2, + 0x157737e5, + 0x58c9290a, + 0xbd5d10f4, + 0xe427418b, + 0xfbee7c66, + 0xca2dbf07, +}; + +__constant u32a rcl[R + 1] = +{ + 0x00000000, + 0x87b8014f, + 0x796f9152, + 0xa30c7b35, + 0x2e4bfe57, + 0x9ff04ada, + 0xb1a06b85, + 0xcb3e0567, + 0xa77d95d8, + 0xdd17479e, + 0xad5a8333, +}; + +// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed + +void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) +{ + u32x Kh[8]; + u32x Kl[8]; + + Kh[0] = 0x300beec0; + Kl[0] = 0xaf902967; + Kh[1] = 0x28282828; + Kl[1] = 0x28282828; + Kh[2] = 0x28282828; + Kl[2] = 0x28282828; + Kh[3] = 0x28282828; + Kl[3] = 0x28282828; + Kh[4] = 0x28282828; + Kl[4] = 0x28282828; + Kh[5] = 0x28282828; + Kl[5] = 0x28282828; + Kh[6] = 0x28282828; + Kl[6] = 0x28282828; + Kh[7] = 0x28282828; + Kl[7] = 0x28282828; + + u32x stateh[8]; + u32x statel[8]; + + stateh[0] = w[ 0]; + statel[0] = w[ 1]; + stateh[1] = w[ 2]; + statel[1] = w[ 3]; + stateh[2] = w[ 4]; + statel[2] = w[ 5]; + stateh[3] = w[ 6]; + statel[3] = w[ 7]; + stateh[4] = w[ 8]; + statel[4] = w[ 9]; + stateh[5] = w[10]; + statel[5] = w[11]; + stateh[6] = w[12]; + statel[6] = w[13]; + stateh[7] = w[14]; + statel[7] = w[15]; + + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + + for (int r = 2; r <= R; r++) + { + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = Kh[(i + 8) & 7] >> 24; + const u32x Lp1 = Kh[(i + 7) & 7] >> 16; + const u32x Lp2 = Kh[(i + 6) & 7] >> 8; + const u32x Lp3 = Kh[(i + 5) & 7] >> 0; + const u32x Lp4 = Kl[(i + 4) & 7] >> 24; + const u32x Lp5 = Kl[(i + 3) & 7] >> 16; + const u32x Lp6 = Kl[(i + 2) & 7] >> 8; + const u32x Lp7 = Kl[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + Kh[0] = Lh[0] ^ rch[r]; + Kl[0] = Ll[0] ^ rcl[r]; + Kh[1] = Lh[1]; + Kl[1] = Ll[1]; + Kh[2] = Lh[2]; + Kl[2] = Ll[2]; + Kh[3] = Lh[3]; + Kl[3] = Ll[3]; + Kh[4] = Lh[4]; + Kl[4] = Ll[4]; + Kh[5] = Lh[5]; + Kl[5] = Ll[5]; + Kh[6] = Lh[6]; + Kl[6] = Ll[6]; + Kh[7] = Lh[7]; + Kl[7] = Ll[7]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + } + + dgst[ 0] = stateh[0] ^ w[ 0]; + dgst[ 1] = statel[0] ^ w[ 1]; + dgst[ 2] = stateh[1] ^ w[ 2]; + dgst[ 3] = statel[1] ^ w[ 3]; + dgst[ 4] = stateh[2] ^ w[ 4]; + dgst[ 5] = statel[2] ^ w[ 5]; + dgst[ 6] = stateh[3] ^ w[ 6]; + dgst[ 7] = statel[3] ^ w[ 7]; + dgst[ 8] = stateh[4] ^ w[ 8]; + dgst[ 9] = statel[4] ^ w[ 9]; + dgst[10] = stateh[5] ^ w[10]; + dgst[11] = statel[5] ^ w[11]; + dgst[12] = stateh[6] ^ w[12]; + dgst[13] = statel[6] ^ w[13]; + dgst[14] = stateh[7] ^ w[14]; + dgst[15] = statel[7] ^ w[15]; +} + +__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + u32x w[16]; + + w[ 0] = swap32 (w0[0]); + w[ 1] = swap32 (w0[1]); + w[ 2] = swap32 (w0[2]); + w[ 3] = swap32 (w0[3]); + w[ 4] = swap32 (w1[0]); + w[ 5] = swap32 (w1[1]); + w[ 6] = swap32 (w1[2]); + w[ 7] = swap32 (w1[3]); + w[ 8] = swap32 (w2[0]); + w[ 9] = swap32 (w2[1]); + w[10] = swap32 (w2[2]); + w[11] = swap32 (w2[3]); + w[12] = swap32 (w3[0]); + w[13] = swap32 (w3[1]); + w[14] = 0; + w[15] = pw_len * 8; + + /** + * Whirlool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + u32x w[16]; + + w[ 0] = swap32 (w0[0]); + w[ 1] = swap32 (w0[1]); + w[ 2] = swap32 (w0[2]); + w[ 3] = swap32 (w0[3]); + w[ 4] = swap32 (w1[0]); + w[ 5] = swap32 (w1[1]); + w[ 6] = swap32 (w1[2]); + w[ 7] = swap32 (w1[3]); + w[ 8] = swap32 (w2[0]); + w[ 9] = swap32 (w2[1]); + w[10] = swap32 (w2[2]); + w[11] = swap32 (w2[3]); + w[12] = swap32 (w3[0]); + w[13] = swap32 (w3[1]); + w[14] = 0; + w[15] = pw_len * 8; + + /** + * Whirlool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m06100_a1-pure.cl b/OpenCL/m06100_a1-pure.cl deleted file mode 100644 index 7311b2c71..000000000 --- a/OpenCL/m06100_a1-pure.cl +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_whirlpool.cl" - -__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - whirlpool_ctx_t ctx0; - - whirlpool_init (&ctx0, s_Ch, s_Cl); - - whirlpool_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - whirlpool_ctx_t ctx = ctx0; - - whirlpool_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - whirlpool_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - whirlpool_ctx_t ctx0; - - whirlpool_init (&ctx0, s_Ch, s_Cl); - - whirlpool_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - whirlpool_ctx_t ctx = ctx0; - - whirlpool_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - whirlpool_final (&ctx); - - const u32 r0 = ctx.h[DGST_R0]; - const u32 r1 = ctx.h[DGST_R1]; - const u32 r2 = ctx.h[DGST_R2]; - const u32 r3 = ctx.h[DGST_R3]; - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m06100_a1.cl b/OpenCL/m06100_a1.cl index 9732930fd..7311b2c71 100644 --- a/OpenCL/m06100_a1.cl +++ b/OpenCL/m06100_a1.cl @@ -3,1345 +3,17 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_whirlpool.cl" -#define R 10 - -#if VECT_SIZE == 1 -#define BOX(S,n,i) (S)[(n)][(i)] -#elif VECT_SIZE == 2 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) -#elif VECT_SIZE == 4 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) -#elif VECT_SIZE == 8 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) -#elif VECT_SIZE == 16 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) -#endif - -__constant u32a Ch[8][256] = -{ - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - } -}; - -__constant u32a Cl[8][256] = -{ - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - }, - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, -}; - -__constant u32a rch[R + 1] = -{ - 0x00000000, - 0x1823c6e8, - 0x36a6d2f5, - 0x60bc9b8e, - 0x1de0d7c2, - 0x157737e5, - 0x58c9290a, - 0xbd5d10f4, - 0xe427418b, - 0xfbee7c66, - 0xca2dbf07, -}; - -__constant u32a rcl[R + 1] = -{ - 0x00000000, - 0x87b8014f, - 0x796f9152, - 0xa30c7b35, - 0x2e4bfe57, - 0x9ff04ada, - 0xb1a06b85, - 0xcb3e0567, - 0xa77d95d8, - 0xdd17479e, - 0xad5a8333, -}; - -// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed - -void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) -{ - u32x Kh[8]; - u32x Kl[8]; - - Kh[0] = 0x300beec0; - Kl[0] = 0xaf902967; - Kh[1] = 0x28282828; - Kl[1] = 0x28282828; - Kh[2] = 0x28282828; - Kl[2] = 0x28282828; - Kh[3] = 0x28282828; - Kl[3] = 0x28282828; - Kh[4] = 0x28282828; - Kl[4] = 0x28282828; - Kh[5] = 0x28282828; - Kl[5] = 0x28282828; - Kh[6] = 0x28282828; - Kl[6] = 0x28282828; - Kh[7] = 0x28282828; - Kl[7] = 0x28282828; - - u32x stateh[8]; - u32x statel[8]; - - stateh[0] = w[ 0]; - statel[0] = w[ 1]; - stateh[1] = w[ 2]; - statel[1] = w[ 3]; - stateh[2] = w[ 4]; - statel[2] = w[ 5]; - stateh[3] = w[ 6]; - statel[3] = w[ 7]; - stateh[4] = w[ 8]; - statel[4] = w[ 9]; - stateh[5] = w[10]; - statel[5] = w[11]; - stateh[6] = w[12]; - statel[6] = w[13]; - stateh[7] = w[14]; - statel[7] = w[15]; - - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - - for (int r = 2; r <= R; r++) - { - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = Kh[(i + 8) & 7] >> 24; - const u32x Lp1 = Kh[(i + 7) & 7] >> 16; - const u32x Lp2 = Kh[(i + 6) & 7] >> 8; - const u32x Lp3 = Kh[(i + 5) & 7] >> 0; - const u32x Lp4 = Kl[(i + 4) & 7] >> 24; - const u32x Lp5 = Kl[(i + 3) & 7] >> 16; - const u32x Lp6 = Kl[(i + 2) & 7] >> 8; - const u32x Lp7 = Kl[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - Kh[0] = Lh[0] ^ rch[r]; - Kl[0] = Ll[0] ^ rcl[r]; - Kh[1] = Lh[1]; - Kl[1] = Ll[1]; - Kh[2] = Lh[2]; - Kl[2] = Ll[2]; - Kh[3] = Lh[3]; - Kl[3] = Ll[3]; - Kh[4] = Lh[4]; - Kl[4] = Ll[4]; - Kh[5] = Lh[5]; - Kl[5] = Ll[5]; - Kh[6] = Lh[6]; - Kl[6] = Ll[6]; - Kh[7] = Lh[7]; - Kl[7] = Ll[7]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - } - - dgst[ 0] = stateh[0] ^ w[ 0]; - dgst[ 1] = statel[0] ^ w[ 1]; - dgst[ 2] = stateh[1] ^ w[ 2]; - dgst[ 3] = statel[1] ^ w[ 3]; - dgst[ 4] = stateh[2] ^ w[ 4]; - dgst[ 5] = statel[2] ^ w[ 5]; - dgst[ 6] = stateh[3] ^ w[ 6]; - dgst[ 7] = statel[3] ^ w[ 7]; - dgst[ 8] = stateh[4] ^ w[ 8]; - dgst[ 9] = statel[4] ^ w[ 9]; - dgst[10] = stateh[5] ^ w[10]; - dgst[11] = statel[5] ^ w[11]; - dgst[12] = stateh[6] ^ w[12]; - dgst[13] = statel[6] ^ w[13]; - dgst[14] = stateh[7] ^ w[14]; - dgst[15] = statel[7] ^ w[15]; -} - -__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1387,150 +59,34 @@ __kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rule * base */ - u32 pw_buf0[4]; - u32 pw_buf1[4]; + whirlpool_ctx_t ctx0; - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; + whirlpool_init (&ctx0, s_Ch, s_Cl); - const u32 pw_l_len = pws[gid].pw_len; + whirlpool_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + whirlpool_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + whirlpool_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + whirlpool_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - u32x w[16]; - - w[ 0] = swap32 (w0[0]); - w[ 1] = swap32 (w0[1]); - w[ 2] = swap32 (w0[2]); - w[ 3] = swap32 (w0[3]); - w[ 4] = swap32 (w1[0]); - w[ 5] = swap32 (w1[1]); - w[ 6] = swap32 (w1[2]); - w[ 7] = swap32 (w1[3]); - w[ 8] = swap32 (w2[0]); - w[ 9] = swap32 (w2[1]); - w[10] = swap32 (w2[2]); - w[11] = swap32 (w2[3]); - w[12] = swap32 (w3[0]); - w[13] = swap32 (w3[1]); - w[14] = 0; - w[15] = pw_len * 8; - - /** - * Whirlool - */ - - u32x dgst[16]; - - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; - - whirlpool_transform (w, dgst, s_Ch, s_Cl); - - COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global netntlm_t *netntlm_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1572,24 +128,6 @@ __kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule if (gid >= gid_max) return; - /** - * base - */ - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -1602,131 +140,33 @@ __kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + whirlpool_ctx_t ctx0; + + whirlpool_init (&ctx0, s_Ch, s_Cl); + + whirlpool_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + whirlpool_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + whirlpool_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + whirlpool_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32 r0 = ctx.h[DGST_R0]; + const u32 r1 = ctx.h[DGST_R1]; + const u32 r2 = ctx.h[DGST_R2]; + const u32 r3 = ctx.h[DGST_R3]; - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - u32x w[16]; - - w[ 0] = swap32 (w0[0]); - w[ 1] = swap32 (w0[1]); - w[ 2] = swap32 (w0[2]); - w[ 3] = swap32 (w0[3]); - w[ 4] = swap32 (w1[0]); - w[ 5] = swap32 (w1[1]); - w[ 6] = swap32 (w1[2]); - w[ 7] = swap32 (w1[3]); - w[ 8] = swap32 (w2[0]); - w[ 9] = swap32 (w2[1]); - w[10] = swap32 (w2[2]); - w[11] = swap32 (w2[3]); - w[12] = swap32 (w3[0]); - w[13] = swap32 (w3[1]); - w[14] = 0; - w[15] = pw_len * 8; - - /** - * Whirlool - */ - - u32x dgst[16]; - - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; - - whirlpool_transform (w, dgst, s_Ch, s_Cl); - - COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m06100_a3-optimized.cl b/OpenCL/m06100_a3-optimized.cl new file mode 100644 index 000000000..12f536664 --- /dev/null +++ b/OpenCL/m06100_a3-optimized.cl @@ -0,0 +1,1990 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +#define R 10 + +#if VECT_SIZE == 1 +#define BOX(S,n,i) (S)[(n)][(i)] +#elif VECT_SIZE == 2 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) +#elif VECT_SIZE == 4 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) +#elif VECT_SIZE == 8 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) +#elif VECT_SIZE == 16 +#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) +#endif + +__constant u32a Ch[8][256] = +{ + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + } +}; + +__constant u32a Cl[8][256] = +{ + { + 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, + 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, + 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, + 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, + 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, + 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, + 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, + 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, + 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, + 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, + 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, + 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, + 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, + 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, + 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, + 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, + 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, + 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, + 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, + 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, + 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, + 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, + 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, + 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, + 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, + 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, + 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, + 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, + 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, + 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, + 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, + 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, + 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, + 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, + 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, + 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, + 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, + 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, + 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, + 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, + 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, + 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, + 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, + 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, + 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, + 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, + 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, + 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, + 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, + 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, + 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, + 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, + 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, + 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, + 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, + 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, + 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, + 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, + 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, + 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, + 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, + 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, + 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, + 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, + }, + { + 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, + 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, + 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, + 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, + 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, + 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, + 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, + 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, + 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, + 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, + 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, + 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, + 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, + 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, + 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, + 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, + 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, + 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, + 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, + 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, + 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, + 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, + 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, + 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, + 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, + 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, + 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, + 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, + 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, + 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, + 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, + 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, + 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, + 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, + 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, + 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, + 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, + 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, + 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, + 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, + 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, + 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, + 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, + 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, + 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, + 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, + 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, + 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, + 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, + 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, + 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, + 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, + 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, + 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, + 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, + 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, + 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, + 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, + 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, + 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, + 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, + 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, + 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, + 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, + }, + { + 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, + 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, + 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, + 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, + 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, + 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, + 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, + 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, + 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, + 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, + 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, + 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, + 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, + 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, + 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, + 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, + 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, + 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, + 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, + 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, + 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, + 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, + 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, + 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, + 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, + 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, + 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, + 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, + 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, + 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, + 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, + 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, + 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, + 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, + 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, + 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, + 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, + 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, + 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, + 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, + 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, + 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, + 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, + 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, + 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, + 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, + 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, + 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, + 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, + 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, + 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, + 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, + 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, + 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, + 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, + 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, + 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, + 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, + 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, + 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, + 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, + 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, + 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, + 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, + }, + { + 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, + 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, + 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, + 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, + 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, + 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, + 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, + 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, + 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, + 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, + 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, + 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, + 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, + 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, + 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, + 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, + 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, + 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, + 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, + 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, + 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, + 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, + 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, + 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, + 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, + 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, + 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, + 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, + 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, + 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, + 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, + 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, + 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, + 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, + 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, + 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, + 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, + 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, + 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, + 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, + 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, + 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, + 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, + 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, + 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, + 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, + 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, + 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, + 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, + 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, + 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, + 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, + 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, + 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, + 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, + 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, + 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, + 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, + 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, + 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, + 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, + 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, + 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, + 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, + }, + { + 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, + 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, + 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, + 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, + 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, + 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, + 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, + 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, + 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, + 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, + 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, + 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, + 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, + 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, + 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, + 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, + 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, + 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, + 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, + 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, + 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, + 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, + 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, + 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, + 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, + 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, + 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, + 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, + 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, + 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, + 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, + 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, + 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, + 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, + 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, + 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, + 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, + 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, + 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, + 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, + 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, + 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, + 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, + 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, + 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, + 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, + 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, + 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, + 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, + 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, + 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, + 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, + 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, + 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, + 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, + 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, + 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, + 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, + 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, + 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, + 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, + 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, + 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, + 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, + }, + { + 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, + 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, + 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, + 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, + 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, + 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, + 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, + 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, + 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, + 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, + 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, + 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, + 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, + 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, + 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, + 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, + 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, + 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, + 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, + 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, + 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, + 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, + 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, + 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, + 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, + 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, + 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, + 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, + 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, + 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, + 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, + 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, + 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, + 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, + 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, + 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, + 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, + 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, + 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, + 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, + 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, + 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, + 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, + 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, + 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, + 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, + 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, + 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, + 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, + 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, + 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, + 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, + 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, + 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, + 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, + 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, + 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, + 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, + 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, + 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, + 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, + 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, + 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, + 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, + }, + { + 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, + 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, + 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, + 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, + 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, + 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, + 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, + 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, + 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, + 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, + 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, + 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, + 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, + 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, + 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, + 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, + 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, + 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, + 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, + 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, + 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, + 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, + 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, + 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, + 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, + 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, + 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, + 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, + 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, + 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, + 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, + 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, + 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, + 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, + 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, + 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, + 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, + 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, + 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, + 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, + 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, + 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, + 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, + 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, + 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, + 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, + 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, + 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, + 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, + 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, + 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, + 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, + 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, + 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, + 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, + 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, + 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, + 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, + 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, + 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, + 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, + 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, + 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, + 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, + }, + { + 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, + 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, + 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, + 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, + 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, + 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, + 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, + 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, + 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, + 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, + 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, + 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, + 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, + 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, + 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, + 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, + 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, + 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, + 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, + 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, + 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, + 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, + 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, + 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, + 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, + 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, + 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, + 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, + 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, + 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, + 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, + 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, + 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, + 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, + 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, + 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, + 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, + 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, + 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, + 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, + 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, + 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, + 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, + 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, + 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, + 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, + 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, + 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, + 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, + 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, + 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, + 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, + 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, + 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, + 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, + 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, + 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, + 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, + 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, + 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, + 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, + 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, + 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, + 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, + }, +}; + +__constant u32a rch[R + 1] = +{ + 0x00000000, + 0x1823c6e8, + 0x36a6d2f5, + 0x60bc9b8e, + 0x1de0d7c2, + 0x157737e5, + 0x58c9290a, + 0xbd5d10f4, + 0xe427418b, + 0xfbee7c66, + 0xca2dbf07, +}; + +__constant u32a rcl[R + 1] = +{ + 0x00000000, + 0x87b8014f, + 0x796f9152, + 0xa30c7b35, + 0x2e4bfe57, + 0x9ff04ada, + 0xb1a06b85, + 0xcb3e0567, + 0xa77d95d8, + 0xdd17479e, + 0xad5a8333, +}; + +// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed + +void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) +{ + u32x Kh[8]; + u32x Kl[8]; + + Kh[0] = 0x300beec0; + Kl[0] = 0xaf902967; + Kh[1] = 0x28282828; + Kl[1] = 0x28282828; + Kh[2] = 0x28282828; + Kl[2] = 0x28282828; + Kh[3] = 0x28282828; + Kl[3] = 0x28282828; + Kh[4] = 0x28282828; + Kl[4] = 0x28282828; + Kh[5] = 0x28282828; + Kl[5] = 0x28282828; + Kh[6] = 0x28282828; + Kl[6] = 0x28282828; + Kh[7] = 0x28282828; + Kl[7] = 0x28282828; + + u32x stateh[8]; + u32x statel[8]; + + stateh[0] = w[ 0]; + statel[0] = w[ 1]; + stateh[1] = w[ 2]; + statel[1] = w[ 3]; + stateh[2] = w[ 4]; + statel[2] = w[ 5]; + stateh[3] = w[ 6]; + statel[3] = w[ 7]; + stateh[4] = w[ 8]; + statel[4] = w[ 9]; + stateh[5] = w[10]; + statel[5] = w[11]; + stateh[6] = w[12]; + statel[6] = w[13]; + stateh[7] = w[14]; + statel[7] = w[15]; + + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + + for (int r = 2; r <= R; r++) + { + u32x Lh[8]; + u32x Ll[8]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = Kh[(i + 8) & 7] >> 24; + const u32x Lp1 = Kh[(i + 7) & 7] >> 16; + const u32x Lp2 = Kh[(i + 6) & 7] >> 8; + const u32x Lp3 = Kh[(i + 5) & 7] >> 0; + const u32x Lp4 = Kl[(i + 4) & 7] >> 24; + const u32x Lp5 = Kl[(i + 3) & 7] >> 16; + const u32x Lp6 = Kl[(i + 2) & 7] >> 8; + const u32x Lp7 = Kl[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + Kh[0] = Lh[0] ^ rch[r]; + Kl[0] = Ll[0] ^ rcl[r]; + Kh[1] = Lh[1]; + Kl[1] = Ll[1]; + Kh[2] = Lh[2]; + Kl[2] = Ll[2]; + Kh[3] = Lh[3]; + Kl[3] = Ll[3]; + Kh[4] = Lh[4]; + Kl[4] = Ll[4]; + Kh[5] = Lh[5]; + Kl[5] = Ll[5]; + Kh[6] = Lh[6]; + Kl[6] = Ll[6]; + Kh[7] = Lh[7]; + Kl[7] = Ll[7]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) + { + const u32x Lp0 = stateh[(i + 8) & 7] >> 24; + const u32x Lp1 = stateh[(i + 7) & 7] >> 16; + const u32x Lp2 = stateh[(i + 6) & 7] >> 8; + const u32x Lp3 = stateh[(i + 5) & 7] >> 0; + const u32x Lp4 = statel[(i + 4) & 7] >> 24; + const u32x Lp5 = statel[(i + 3) & 7] >> 16; + const u32x Lp6 = statel[(i + 2) & 7] >> 8; + const u32x Lp7 = statel[(i + 1) & 7] >> 0; + + Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) + ^ BOX (s_Ch, 1, Lp1 & 0xff) + ^ BOX (s_Ch, 2, Lp2 & 0xff) + ^ BOX (s_Ch, 3, Lp3 & 0xff) + ^ BOX (s_Ch, 4, Lp4 & 0xff) + ^ BOX (s_Ch, 5, Lp5 & 0xff) + ^ BOX (s_Ch, 6, Lp6 & 0xff) + ^ BOX (s_Ch, 7, Lp7 & 0xff); + + Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) + ^ BOX (s_Cl, 1, Lp1 & 0xff) + ^ BOX (s_Cl, 2, Lp2 & 0xff) + ^ BOX (s_Cl, 3, Lp3 & 0xff) + ^ BOX (s_Cl, 4, Lp4 & 0xff) + ^ BOX (s_Cl, 5, Lp5 & 0xff) + ^ BOX (s_Cl, 6, Lp6 & 0xff) + ^ BOX (s_Cl, 7, Lp7 & 0xff); + } + + stateh[0] = Lh[0] ^ Kh[0]; + statel[0] = Ll[0] ^ Kl[0]; + stateh[1] = Lh[1] ^ Kh[1]; + statel[1] = Ll[1] ^ Kl[1]; + stateh[2] = Lh[2] ^ Kh[2]; + statel[2] = Ll[2] ^ Kl[2]; + stateh[3] = Lh[3] ^ Kh[3]; + statel[3] = Ll[3] ^ Kl[3]; + stateh[4] = Lh[4] ^ Kh[4]; + statel[4] = Ll[4] ^ Kl[4]; + stateh[5] = Lh[5] ^ Kh[5]; + statel[5] = Ll[5] ^ Kl[5]; + stateh[6] = Lh[6] ^ Kh[6]; + statel[6] = Ll[6] ^ Kl[6]; + stateh[7] = Lh[7] ^ Kh[7]; + statel[7] = Ll[7] ^ Kl[7]; + } + + dgst[ 0] = stateh[0] ^ w[ 0]; + dgst[ 1] = statel[0] ^ w[ 1]; + dgst[ 2] = stateh[1] ^ w[ 2]; + dgst[ 3] = statel[1] ^ w[ 3]; + dgst[ 4] = stateh[2] ^ w[ 4]; + dgst[ 5] = statel[2] ^ w[ 5]; + dgst[ 6] = stateh[3] ^ w[ 6]; + dgst[ 7] = statel[3] ^ w[ 7]; + dgst[ 8] = stateh[4] ^ w[ 8]; + dgst[ 9] = statel[4] ^ w[ 9]; + dgst[10] = stateh[5] ^ w[10]; + dgst[11] = statel[5] ^ w[11]; + dgst[12] = stateh[6] ^ w[12]; + dgst[13] = statel[6] ^ w[13]; + dgst[14] = stateh[7] ^ w[14]; + dgst[15] = statel[7] ^ w[15]; +} + +void m06100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 (*s_Cl)[256], __local u32 (*s_Ch)[256]) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w[16]; + + w[ 0] = w0lr; + w[ 1] = w0[1]; + w[ 2] = w0[2]; + w[ 3] = w0[3]; + w[ 4] = w1[0]; + w[ 5] = w1[1]; + w[ 6] = w1[2]; + w[ 7] = w1[3]; + w[ 8] = w2[0]; + w[ 9] = w2[1]; + w[10] = w2[2]; + w[11] = w2[3]; + w[12] = w3[0]; + w[13] = w3[1]; + w[14] = 0; + w[15] = pw_len * 8; + + /** + * Whirlool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +void m06100s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 (*s_Cl)[256], __local u32 (*s_Ch)[256]) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w0[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = ix_create_bft (bfs_buf, il_pos); + + const u32x w0lr = w0l | w0r; + + u32x w[16]; + + w[ 0] = w0lr; + w[ 1] = w0[1]; + w[ 2] = w0[2]; + w[ 3] = w0[3]; + w[ 4] = w1[0]; + w[ 5] = w1[1]; + w[ 6] = w1[2]; + w[ 7] = w1[3]; + w[ 8] = w2[0]; + w[ 9] = w2[1]; + w[10] = w2[2]; + w[11] = w2[3]; + w[12] = w3[0]; + w[13] = w3[1]; + w[14] = 0; + w[15] = pw_len * 8; + + /** + * Whirlool + */ + + u32x dgst[16]; + + dgst[ 0] = 0; + dgst[ 1] = 0; + dgst[ 2] = 0; + dgst[ 3] = 0; + dgst[ 4] = 0; + dgst[ 5] = 0; + dgst[ 6] = 0; + dgst[ 7] = 0; + dgst[ 8] = 0; + dgst[ 9] = 0; + dgst[10] = 0; + dgst[11] = 0; + dgst[12] = 0; + dgst[13] = 0; + dgst[14] = 0; + dgst[15] = 0; + + whirlpool_transform (w, dgst, s_Ch, s_Cl); + + COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + } +} + +__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} + +__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} + +__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} + +__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = 0; + w1[1] = 0; + w1[2] = 0; + w1[3] = 0; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} + +__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + + u32 w3[4]; + + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} + +__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[ 0]; + w0[1] = pws[gid].i[ 1]; + w0[2] = pws[gid].i[ 2]; + w0[3] = pws[gid].i[ 3]; + + u32 w1[4]; + + w1[0] = pws[gid].i[ 4]; + w1[1] = pws[gid].i[ 5]; + w1[2] = pws[gid].i[ 6]; + w1[3] = pws[gid].i[ 7]; + + u32 w2[4]; + + w2[0] = pws[gid].i[ 8]; + w2[1] = pws[gid].i[ 9]; + w2[2] = pws[gid].i[10]; + w2[3] = pws[gid].i[11]; + + u32 w3[4]; + + w3[0] = pws[gid].i[12]; + w3[1] = pws[gid].i[13]; + w3[2] = 0; + w3[3] = 0; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); +} diff --git a/OpenCL/m06100_a3-pure.cl b/OpenCL/m06100_a3-pure.cl deleted file mode 100644 index da99bac26..000000000 --- a/OpenCL/m06100_a3-pure.cl +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_whirlpool.cl" - -__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - whirlpool_ctx_vector_t ctx; - - whirlpool_init_vector (&ctx, s_Ch, s_Cl); - - whirlpool_update_vector (&ctx, w, pw_len); - - whirlpool_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - whirlpool_ctx_vector_t ctx; - - whirlpool_init_vector (&ctx, s_Ch, s_Cl); - - whirlpool_update_vector (&ctx, w, pw_len); - - whirlpool_final_vector (&ctx); - - const u32x r0 = ctx.h[DGST_R0]; - const u32x r1 = ctx.h[DGST_R1]; - const u32x r2 = ctx.h[DGST_R2]; - const u32x r3 = ctx.h[DGST_R3]; - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m06100_a3.cl b/OpenCL/m06100_a3.cl index 12f536664..da99bac26 100644 --- a/OpenCL/m06100_a3.cl +++ b/OpenCL/m06100_a3.cl @@ -11,1337 +11,9 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_whirlpool.cl" -#define R 10 - -#if VECT_SIZE == 1 -#define BOX(S,n,i) (S)[(n)][(i)] -#elif VECT_SIZE == 2 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1]) -#elif VECT_SIZE == 4 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3]) -#elif VECT_SIZE == 8 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7]) -#elif VECT_SIZE == 16 -#define BOX(S,n,i) (u32x) ((S)[(n)][(i).s0], (S)[(n)][(i).s1], (S)[(n)][(i).s2], (S)[(n)][(i).s3], (S)[(n)][(i).s4], (S)[(n)][(i).s5], (S)[(n)][(i).s6], (S)[(n)][(i).s7], (S)[(n)][(i).s8], (S)[(n)][(i).s9], (S)[(n)][(i).sa], (S)[(n)][(i).sb], (S)[(n)][(i).sc], (S)[(n)][(i).sd], (S)[(n)][(i).se], (S)[(n)][(i).sf]) -#endif - -__constant u32a Ch[8][256] = -{ - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - } -}; - -__constant u32a Cl[8][256] = -{ - { - 0xc07830d8, 0x05af4626, 0x7ef991b8, 0x136fcdfb, - 0x4ca113cb, 0xa9626d11, 0x08050209, 0x426e9e0d, - 0xadee6c9b, 0x590451ff, 0xdebdb90c, 0xfb06f70e, - 0xef80f296, 0x5fcede30, 0xfcef3f6d, 0xaa07a4f8, - 0x27fdc047, 0x89766535, 0xaccd2b37, 0x048c018a, - 0x71155bd2, 0x603c186c, 0xff8af684, 0xb5e16a80, - 0xe8693af5, 0x5347ddb3, 0xf6acb321, 0x5eed999c, - 0x6d965c43, 0x627a9629, 0xa321e15d, 0x8216aed5, - 0xa8412abd, 0x9fb6eee8, 0xa5eb6e92, 0x7b56d79e, - 0x8cd92313, 0xd317fd23, 0x6a7f9420, 0x9e95a944, - 0xfa25b0a2, 0x06ca8fcf, 0x558d527c, 0x5022145a, - 0xe14f7f50, 0x691a5dc9, 0x7fdad614, 0x5cab17d9, - 0x8173673c, 0xd234ba8f, 0x80502090, 0xf303f507, - 0x16c08bdd, 0xedc67cd3, 0x28110a2d, 0x1fe6ce78, - 0x7353d597, 0x25bb4e02, 0x32588273, 0x2c9d0ba7, - 0x510153f6, 0xcf94fab2, 0xdcfb3749, 0x8e9fad56, - 0x8b30eb70, 0x2371c1cd, 0xc791f8bb, 0x17e3cc71, - 0xa68ea77b, 0xb84b2eaf, 0x02468e45, 0x84dc211a, - 0x1ec589d4, 0x75995a58, 0x9179632e, 0x381b0e3f, - 0x012347ac, 0xea2fb4b0, 0x6cb51bef, 0x85ff66b6, - 0x3ff2c65c, 0x100a0412, 0x39384993, 0xafa8e2de, - 0x0ecf8dc6, 0xc87d32d1, 0x7270923b, 0x869aaf5f, - 0xc31df931, 0x4b48dba8, 0xe22ab6b9, 0x34920dbc, - 0xa4c8293e, 0x2dbe4c0b, 0x8dfa64bf, 0xe94a7d59, - 0x1b6acff2, 0x78331e77, 0xe6a6b733, 0x74ba1df4, - 0x997c6127, 0x26de87eb, 0xbde46889, 0x7a759032, - 0xab24e354, 0xf78ff48d, 0xf4ea3d64, 0xc23ebe9d, - 0x1da0403d, 0x67d5d00f, 0xd07234ca, 0x192c41b7, - 0xc95e757d, 0x9a19a8ce, 0xece53b7f, 0x0daa442f, - 0x07e9c863, 0xdb12ff2a, 0xbfa2e6cc, 0x905a2482, - 0x3a5d807a, 0x40281048, 0x56e89b95, 0x337bc5df, - 0x9690ab4d, 0x611f5fc0, 0x1c830791, 0xf5c97ac8, - 0xccf1335b, 0x00000000, 0x36d483f9, 0x4587566e, - 0x97b3ece1, 0x64b019e6, 0xfea9b128, 0xd87736c3, - 0xc15b7774, 0x112943be, 0x77dfd41d, 0xba0da0ea, - 0x124c8a57, 0xcb18fb38, 0x9df060ad, 0x2b74c3c4, - 0xe5c37eda, 0x921caac7, 0x791059db, 0x0365c9e9, - 0x0fecca6a, 0xb9686903, 0x65935e4a, 0x4ee79d8e, - 0xbe81a160, 0xe06c38fc, 0xbb2ee746, 0x52649a1f, - 0xe4e03976, 0x8fbceafa, 0x301e0c36, 0x249809ae, - 0xf940794b, 0x6359d185, 0x70361c7e, 0xf8633ee7, - 0x37f7c455, 0xeea3b53a, 0x29324d81, 0xc4f43152, - 0x9b3aef62, 0x66f697a3, 0x35b14a10, 0xf220b2ab, - 0x54ae15d0, 0xb7a7e4c5, 0xd5dd72ec, 0x5a619816, - 0xca3bbc94, 0xe785f09f, 0xddd870e5, 0x14860598, - 0xc6b2bf17, 0x410b57e4, 0x434dd9a1, 0x2ff8c24e, - 0xf1457b42, 0x15a54234, 0x94d62508, 0xf0663cee, - 0x22528661, 0x76fc93b1, 0xb32be54f, 0x20140824, - 0xb208a2e3, 0xbcc72f25, 0x4fc4da22, 0x68391a65, - 0x8335e979, 0xb684a369, 0xd79bfca9, 0x3db44819, - 0xc5d776fe, 0x313d4b9a, 0x3ed181f0, 0x88552299, - 0x0c890383, 0x4a6b9c04, 0xd1517366, 0x0b60cbe0, - 0xfdcc78c1, 0x7cbf1ffd, 0xd4fe3540, 0xeb0cf31c, - 0xa1676f18, 0x985f268b, 0x7d9c5851, 0xd6b8bb05, - 0x6b5cd38c, 0x57cbdc39, 0x6ef395aa, 0x180f061b, - 0x8a13acdc, 0x1a49885e, 0xdf9efea0, 0x21374f88, - 0x4d825467, 0xb16d6b0a, 0x46e29f87, 0xa202a6f1, - 0xae8ba572, 0x58271653, 0x9cd32701, 0x47c1d82b, - 0x95f562a4, 0x87b9e8f3, 0xe309f115, 0x0a438c4c, - 0x092645a5, 0x3c970fb5, 0xa04428b4, 0x5b42dfba, - 0xb04e2ca6, 0xcdd274f7, 0x6fd0d206, 0x482d1241, - 0xa7ade0d7, 0xd954716f, 0xceb7bd1e, 0x3b7ec7d6, - 0x2edb85e2, 0x2a578468, 0xb4c22d2c, 0x490e55ed, - 0x5d885075, 0xda31b886, 0x933fed6b, 0x44a411c2, - }, - { - 0x18c07830, 0x2305af46, 0xc67ef991, 0xe8136fcd, - 0x874ca113, 0xb8a9626d, 0x01080502, 0x4f426e9e, - 0x36adee6c, 0xa6590451, 0xd2debdb9, 0xf5fb06f7, - 0x79ef80f2, 0x6f5fcede, 0x91fcef3f, 0x52aa07a4, - 0x6027fdc0, 0xbc897665, 0x9baccd2b, 0x8e048c01, - 0xa371155b, 0x0c603c18, 0x7bff8af6, 0x35b5e16a, - 0x1de8693a, 0xe05347dd, 0xd7f6acb3, 0xc25eed99, - 0x2e6d965c, 0x4b627a96, 0xfea321e1, 0x578216ae, - 0x15a8412a, 0x779fb6ee, 0x37a5eb6e, 0xe57b56d7, - 0x9f8cd923, 0xf0d317fd, 0x4a6a7f94, 0xda9e95a9, - 0x58fa25b0, 0xc906ca8f, 0x29558d52, 0x0a502214, - 0xb1e14f7f, 0xa0691a5d, 0x6b7fdad6, 0x855cab17, - 0xbd817367, 0x5dd234ba, 0x10805020, 0xf4f303f5, - 0xcb16c08b, 0x3eedc67c, 0x0528110a, 0x671fe6ce, - 0xe47353d5, 0x2725bb4e, 0x41325882, 0x8b2c9d0b, - 0xa7510153, 0x7dcf94fa, 0x95dcfb37, 0xd88e9fad, - 0xfb8b30eb, 0xee2371c1, 0x7cc791f8, 0x6617e3cc, - 0xdda68ea7, 0x17b84b2e, 0x4702468e, 0x9e84dc21, - 0xca1ec589, 0x2d75995a, 0xbf917963, 0x07381b0e, - 0xad012347, 0x5aea2fb4, 0x836cb51b, 0x3385ff66, - 0x633ff2c6, 0x02100a04, 0xaa393849, 0x71afa8e2, - 0xc80ecf8d, 0x19c87d32, 0x49727092, 0xd9869aaf, - 0xf2c31df9, 0xe34b48db, 0x5be22ab6, 0x8834920d, - 0x9aa4c829, 0x262dbe4c, 0x328dfa64, 0xb0e94a7d, - 0xe91b6acf, 0x0f78331e, 0xd5e6a6b7, 0x8074ba1d, - 0xbe997c61, 0xcd26de87, 0x34bde468, 0x487a7590, - 0xffab24e3, 0x7af78ff4, 0x90f4ea3d, 0x5fc23ebe, - 0x201da040, 0x6867d5d0, 0x1ad07234, 0xae192c41, - 0xb4c95e75, 0x549a19a8, 0x93ece53b, 0x220daa44, - 0x6407e9c8, 0xf1db12ff, 0x73bfa2e6, 0x12905a24, - 0x403a5d80, 0x08402810, 0xc356e89b, 0xec337bc5, - 0xdb9690ab, 0xa1611f5f, 0x8d1c8307, 0x3df5c97a, - 0x97ccf133, 0x00000000, 0xcf36d483, 0x2b458756, - 0x7697b3ec, 0x8264b019, 0xd6fea9b1, 0x1bd87736, - 0xb5c15b77, 0xaf112943, 0x6a77dfd4, 0x50ba0da0, - 0x45124c8a, 0xf3cb18fb, 0x309df060, 0xef2b74c3, - 0x3fe5c37e, 0x55921caa, 0xa2791059, 0xea0365c9, - 0x650fecca, 0xbab96869, 0x2f65935e, 0xc04ee79d, - 0xdebe81a1, 0x1ce06c38, 0xfdbb2ee7, 0x4d52649a, - 0x92e4e039, 0x758fbcea, 0x06301e0c, 0x8a249809, - 0xb2f94079, 0xe66359d1, 0x0e70361c, 0x1ff8633e, - 0x6237f7c4, 0xd4eea3b5, 0xa829324d, 0x96c4f431, - 0xf99b3aef, 0xc566f697, 0x2535b14a, 0x59f220b2, - 0x8454ae15, 0x72b7a7e4, 0x39d5dd72, 0x4c5a6198, - 0x5eca3bbc, 0x78e785f0, 0x38ddd870, 0x8c148605, - 0xd1c6b2bf, 0xa5410b57, 0xe2434dd9, 0x612ff8c2, - 0xb3f1457b, 0x2115a542, 0x9c94d625, 0x1ef0663c, - 0x43225286, 0xc776fc93, 0xfcb32be5, 0x04201408, - 0x51b208a2, 0x99bcc72f, 0x6d4fc4da, 0x0d68391a, - 0xfa8335e9, 0xdfb684a3, 0x7ed79bfc, 0x243db448, - 0x3bc5d776, 0xab313d4b, 0xce3ed181, 0x11885522, - 0x8f0c8903, 0x4e4a6b9c, 0xb7d15173, 0xeb0b60cb, - 0x3cfdcc78, 0x817cbf1f, 0x94d4fe35, 0xf7eb0cf3, - 0xb9a1676f, 0x13985f26, 0x2c7d9c58, 0xd3d6b8bb, - 0xe76b5cd3, 0x6e57cbdc, 0xc46ef395, 0x03180f06, - 0x568a13ac, 0x441a4988, 0x7fdf9efe, 0xa921374f, - 0x2a4d8254, 0xbbb16d6b, 0xc146e29f, 0x53a202a6, - 0xdcae8ba5, 0x0b582716, 0x9d9cd327, 0x6c47c1d8, - 0x3195f562, 0x7487b9e8, 0xf6e309f1, 0x460a438c, - 0xac092645, 0x893c970f, 0x14a04428, 0xe15b42df, - 0x16b04e2c, 0x3acdd274, 0x696fd0d2, 0x09482d12, - 0x70a7ade0, 0xb6d95471, 0xd0ceb7bd, 0xed3b7ec7, - 0xcc2edb85, 0x422a5784, 0x98b4c22d, 0xa4490e55, - 0x285d8850, 0x5cda31b8, 0xf8933fed, 0x8644a411, - }, - { - 0x6018c078, 0x8c2305af, 0x3fc67ef9, 0x87e8136f, - 0x26874ca1, 0xdab8a962, 0x04010805, 0x214f426e, - 0xd836adee, 0xa2a65904, 0x6fd2debd, 0xf3f5fb06, - 0xf979ef80, 0xa16f5fce, 0x7e91fcef, 0x5552aa07, - 0x9d6027fd, 0xcabc8976, 0x569baccd, 0x028e048c, - 0xb6a37115, 0x300c603c, 0xf17bff8a, 0xd435b5e1, - 0x741de869, 0xa7e05347, 0x7bd7f6ac, 0x2fc25eed, - 0xb82e6d96, 0x314b627a, 0xdffea321, 0x41578216, - 0x5415a841, 0xc1779fb6, 0xdc37a5eb, 0xb3e57b56, - 0x469f8cd9, 0xe7f0d317, 0x354a6a7f, 0x4fda9e95, - 0x7d58fa25, 0x03c906ca, 0xa429558d, 0x280a5022, - 0xfeb1e14f, 0xbaa0691a, 0xb16b7fda, 0x2e855cab, - 0xcebd8173, 0x695dd234, 0x40108050, 0xf7f4f303, - 0x0bcb16c0, 0xf83eedc6, 0x14052811, 0x81671fe6, - 0xb7e47353, 0x9c2725bb, 0x19413258, 0x168b2c9d, - 0xa6a75101, 0xe97dcf94, 0x6e95dcfb, 0x47d88e9f, - 0xcbfb8b30, 0x9fee2371, 0xed7cc791, 0x856617e3, - 0x53dda68e, 0x5c17b84b, 0x01470246, 0x429e84dc, - 0x0fca1ec5, 0xb42d7599, 0xc6bf9179, 0x1c07381b, - 0x8ead0123, 0x755aea2f, 0x36836cb5, 0xcc3385ff, - 0x91633ff2, 0x0802100a, 0x92aa3938, 0xd971afa8, - 0x07c80ecf, 0x6419c87d, 0x39497270, 0x43d9869a, - 0xeff2c31d, 0xabe34b48, 0x715be22a, 0x1a883492, - 0x529aa4c8, 0x98262dbe, 0xc8328dfa, 0xfab0e94a, - 0x83e91b6a, 0x3c0f7833, 0x73d5e6a6, 0x3a8074ba, - 0xc2be997c, 0x13cd26de, 0xd034bde4, 0x3d487a75, - 0xdbffab24, 0xf57af78f, 0x7a90f4ea, 0x615fc23e, - 0x80201da0, 0xbd6867d5, 0x681ad072, 0x82ae192c, - 0xeab4c95e, 0x4d549a19, 0x7693ece5, 0x88220daa, - 0x8d6407e9, 0xe3f1db12, 0xd173bfa2, 0x4812905a, - 0x1d403a5d, 0x20084028, 0x2bc356e8, 0x97ec337b, - 0x4bdb9690, 0xbea1611f, 0x0e8d1c83, 0xf43df5c9, - 0x6697ccf1, 0x00000000, 0x1bcf36d4, 0xac2b4587, - 0xc57697b3, 0x328264b0, 0x7fd6fea9, 0x6c1bd877, - 0xeeb5c15b, 0x86af1129, 0xb56a77df, 0x5d50ba0d, - 0x0945124c, 0xebf3cb18, 0xc0309df0, 0x9bef2b74, - 0xfc3fe5c3, 0x4955921c, 0xb2a27910, 0x8fea0365, - 0x89650fec, 0xd2bab968, 0xbc2f6593, 0x27c04ee7, - 0x5fdebe81, 0x701ce06c, 0xd3fdbb2e, 0x294d5264, - 0x7292e4e0, 0xc9758fbc, 0x1806301e, 0x128a2498, - 0xf2b2f940, 0xbfe66359, 0x380e7036, 0x7c1ff863, - 0x956237f7, 0x77d4eea3, 0x9aa82932, 0x6296c4f4, - 0xc3f99b3a, 0x33c566f6, 0x942535b1, 0x7959f220, - 0x2a8454ae, 0xd572b7a7, 0xe439d5dd, 0x2d4c5a61, - 0x655eca3b, 0xfd78e785, 0xe038ddd8, 0x0a8c1486, - 0x63d1c6b2, 0xaea5410b, 0xafe2434d, 0x99612ff8, - 0xf6b3f145, 0x842115a5, 0x4a9c94d6, 0x781ef066, - 0x11432252, 0x3bc776fc, 0xd7fcb32b, 0x10042014, - 0x5951b208, 0x5e99bcc7, 0xa96d4fc4, 0x340d6839, - 0xcffa8335, 0x5bdfb684, 0xe57ed79b, 0x90243db4, - 0xec3bc5d7, 0x96ab313d, 0x1fce3ed1, 0x44118855, - 0x068f0c89, 0x254e4a6b, 0xe6b7d151, 0x8beb0b60, - 0xf03cfdcc, 0x3e817cbf, 0x6a94d4fe, 0xfbf7eb0c, - 0xdeb9a167, 0x4c13985f, 0xb02c7d9c, 0x6bd3d6b8, - 0xbbe76b5c, 0xa56e57cb, 0x37c46ef3, 0x0c03180f, - 0x45568a13, 0x0d441a49, 0xe17fdf9e, 0x9ea92137, - 0xa82a4d82, 0xd6bbb16d, 0x23c146e2, 0x5153a202, - 0x57dcae8b, 0x2c0b5827, 0x4e9d9cd3, 0xad6c47c1, - 0xc43195f5, 0xcd7487b9, 0xfff6e309, 0x05460a43, - 0x8aac0926, 0x1e893c97, 0x5014a044, 0xa3e15b42, - 0x5816b04e, 0xe83acdd2, 0xb9696fd0, 0x2409482d, - 0xdd70a7ad, 0xe2b6d954, 0x67d0ceb7, 0x93ed3b7e, - 0x17cc2edb, 0x15422a57, 0x5a98b4c2, 0xaaa4490e, - 0xa0285d88, 0x6d5cda31, 0xc7f8933f, 0x228644a4, - }, - { - 0x186018c0, 0x238c2305, 0xc63fc67e, 0xe887e813, - 0x8726874c, 0xb8dab8a9, 0x01040108, 0x4f214f42, - 0x36d836ad, 0xa6a2a659, 0xd26fd2de, 0xf5f3f5fb, - 0x79f979ef, 0x6fa16f5f, 0x917e91fc, 0x525552aa, - 0x609d6027, 0xbccabc89, 0x9b569bac, 0x8e028e04, - 0xa3b6a371, 0x0c300c60, 0x7bf17bff, 0x35d435b5, - 0x1d741de8, 0xe0a7e053, 0xd77bd7f6, 0xc22fc25e, - 0x2eb82e6d, 0x4b314b62, 0xfedffea3, 0x57415782, - 0x155415a8, 0x77c1779f, 0x37dc37a5, 0xe5b3e57b, - 0x9f469f8c, 0xf0e7f0d3, 0x4a354a6a, 0xda4fda9e, - 0x587d58fa, 0xc903c906, 0x29a42955, 0x0a280a50, - 0xb1feb1e1, 0xa0baa069, 0x6bb16b7f, 0x852e855c, - 0xbdcebd81, 0x5d695dd2, 0x10401080, 0xf4f7f4f3, - 0xcb0bcb16, 0x3ef83eed, 0x05140528, 0x6781671f, - 0xe4b7e473, 0x279c2725, 0x41194132, 0x8b168b2c, - 0xa7a6a751, 0x7de97dcf, 0x956e95dc, 0xd847d88e, - 0xfbcbfb8b, 0xee9fee23, 0x7ced7cc7, 0x66856617, - 0xdd53dda6, 0x175c17b8, 0x47014702, 0x9e429e84, - 0xca0fca1e, 0x2db42d75, 0xbfc6bf91, 0x071c0738, - 0xad8ead01, 0x5a755aea, 0x8336836c, 0x33cc3385, - 0x6391633f, 0x02080210, 0xaa92aa39, 0x71d971af, - 0xc807c80e, 0x196419c8, 0x49394972, 0xd943d986, - 0xf2eff2c3, 0xe3abe34b, 0x5b715be2, 0x881a8834, - 0x9a529aa4, 0x2698262d, 0x32c8328d, 0xb0fab0e9, - 0xe983e91b, 0x0f3c0f78, 0xd573d5e6, 0x803a8074, - 0xbec2be99, 0xcd13cd26, 0x34d034bd, 0x483d487a, - 0xffdbffab, 0x7af57af7, 0x907a90f4, 0x5f615fc2, - 0x2080201d, 0x68bd6867, 0x1a681ad0, 0xae82ae19, - 0xb4eab4c9, 0x544d549a, 0x937693ec, 0x2288220d, - 0x648d6407, 0xf1e3f1db, 0x73d173bf, 0x12481290, - 0x401d403a, 0x08200840, 0xc32bc356, 0xec97ec33, - 0xdb4bdb96, 0xa1bea161, 0x8d0e8d1c, 0x3df43df5, - 0x976697cc, 0x00000000, 0xcf1bcf36, 0x2bac2b45, - 0x76c57697, 0x82328264, 0xd67fd6fe, 0x1b6c1bd8, - 0xb5eeb5c1, 0xaf86af11, 0x6ab56a77, 0x505d50ba, - 0x45094512, 0xf3ebf3cb, 0x30c0309d, 0xef9bef2b, - 0x3ffc3fe5, 0x55495592, 0xa2b2a279, 0xea8fea03, - 0x6589650f, 0xbad2bab9, 0x2fbc2f65, 0xc027c04e, - 0xde5fdebe, 0x1c701ce0, 0xfdd3fdbb, 0x4d294d52, - 0x927292e4, 0x75c9758f, 0x06180630, 0x8a128a24, - 0xb2f2b2f9, 0xe6bfe663, 0x0e380e70, 0x1f7c1ff8, - 0x62956237, 0xd477d4ee, 0xa89aa829, 0x966296c4, - 0xf9c3f99b, 0xc533c566, 0x25942535, 0x597959f2, - 0x842a8454, 0x72d572b7, 0x39e439d5, 0x4c2d4c5a, - 0x5e655eca, 0x78fd78e7, 0x38e038dd, 0x8c0a8c14, - 0xd163d1c6, 0xa5aea541, 0xe2afe243, 0x6199612f, - 0xb3f6b3f1, 0x21842115, 0x9c4a9c94, 0x1e781ef0, - 0x43114322, 0xc73bc776, 0xfcd7fcb3, 0x04100420, - 0x515951b2, 0x995e99bc, 0x6da96d4f, 0x0d340d68, - 0xfacffa83, 0xdf5bdfb6, 0x7ee57ed7, 0x2490243d, - 0x3bec3bc5, 0xab96ab31, 0xce1fce3e, 0x11441188, - 0x8f068f0c, 0x4e254e4a, 0xb7e6b7d1, 0xeb8beb0b, - 0x3cf03cfd, 0x813e817c, 0x946a94d4, 0xf7fbf7eb, - 0xb9deb9a1, 0x134c1398, 0x2cb02c7d, 0xd36bd3d6, - 0xe7bbe76b, 0x6ea56e57, 0xc437c46e, 0x030c0318, - 0x5645568a, 0x440d441a, 0x7fe17fdf, 0xa99ea921, - 0x2aa82a4d, 0xbbd6bbb1, 0xc123c146, 0x535153a2, - 0xdc57dcae, 0x0b2c0b58, 0x9d4e9d9c, 0x6cad6c47, - 0x31c43195, 0x74cd7487, 0xf6fff6e3, 0x4605460a, - 0xac8aac09, 0x891e893c, 0x145014a0, 0xe1a3e15b, - 0x165816b0, 0x3ae83acd, 0x69b9696f, 0x09240948, - 0x70dd70a7, 0xb6e2b6d9, 0xd067d0ce, 0xed93ed3b, - 0xcc17cc2e, 0x4215422a, 0x985a98b4, 0xa4aaa449, - 0x28a0285d, 0x5c6d5cda, 0xf8c7f893, 0x86228644, - }, - { - 0x18186018, 0x23238c23, 0xc6c63fc6, 0xe8e887e8, - 0x87872687, 0xb8b8dab8, 0x01010401, 0x4f4f214f, - 0x3636d836, 0xa6a6a2a6, 0xd2d26fd2, 0xf5f5f3f5, - 0x7979f979, 0x6f6fa16f, 0x91917e91, 0x52525552, - 0x60609d60, 0xbcbccabc, 0x9b9b569b, 0x8e8e028e, - 0xa3a3b6a3, 0x0c0c300c, 0x7b7bf17b, 0x3535d435, - 0x1d1d741d, 0xe0e0a7e0, 0xd7d77bd7, 0xc2c22fc2, - 0x2e2eb82e, 0x4b4b314b, 0xfefedffe, 0x57574157, - 0x15155415, 0x7777c177, 0x3737dc37, 0xe5e5b3e5, - 0x9f9f469f, 0xf0f0e7f0, 0x4a4a354a, 0xdada4fda, - 0x58587d58, 0xc9c903c9, 0x2929a429, 0x0a0a280a, - 0xb1b1feb1, 0xa0a0baa0, 0x6b6bb16b, 0x85852e85, - 0xbdbdcebd, 0x5d5d695d, 0x10104010, 0xf4f4f7f4, - 0xcbcb0bcb, 0x3e3ef83e, 0x05051405, 0x67678167, - 0xe4e4b7e4, 0x27279c27, 0x41411941, 0x8b8b168b, - 0xa7a7a6a7, 0x7d7de97d, 0x95956e95, 0xd8d847d8, - 0xfbfbcbfb, 0xeeee9fee, 0x7c7ced7c, 0x66668566, - 0xdddd53dd, 0x17175c17, 0x47470147, 0x9e9e429e, - 0xcaca0fca, 0x2d2db42d, 0xbfbfc6bf, 0x07071c07, - 0xadad8ead, 0x5a5a755a, 0x83833683, 0x3333cc33, - 0x63639163, 0x02020802, 0xaaaa92aa, 0x7171d971, - 0xc8c807c8, 0x19196419, 0x49493949, 0xd9d943d9, - 0xf2f2eff2, 0xe3e3abe3, 0x5b5b715b, 0x88881a88, - 0x9a9a529a, 0x26269826, 0x3232c832, 0xb0b0fab0, - 0xe9e983e9, 0x0f0f3c0f, 0xd5d573d5, 0x80803a80, - 0xbebec2be, 0xcdcd13cd, 0x3434d034, 0x48483d48, - 0xffffdbff, 0x7a7af57a, 0x90907a90, 0x5f5f615f, - 0x20208020, 0x6868bd68, 0x1a1a681a, 0xaeae82ae, - 0xb4b4eab4, 0x54544d54, 0x93937693, 0x22228822, - 0x64648d64, 0xf1f1e3f1, 0x7373d173, 0x12124812, - 0x40401d40, 0x08082008, 0xc3c32bc3, 0xecec97ec, - 0xdbdb4bdb, 0xa1a1bea1, 0x8d8d0e8d, 0x3d3df43d, - 0x97976697, 0x00000000, 0xcfcf1bcf, 0x2b2bac2b, - 0x7676c576, 0x82823282, 0xd6d67fd6, 0x1b1b6c1b, - 0xb5b5eeb5, 0xafaf86af, 0x6a6ab56a, 0x50505d50, - 0x45450945, 0xf3f3ebf3, 0x3030c030, 0xefef9bef, - 0x3f3ffc3f, 0x55554955, 0xa2a2b2a2, 0xeaea8fea, - 0x65658965, 0xbabad2ba, 0x2f2fbc2f, 0xc0c027c0, - 0xdede5fde, 0x1c1c701c, 0xfdfdd3fd, 0x4d4d294d, - 0x92927292, 0x7575c975, 0x06061806, 0x8a8a128a, - 0xb2b2f2b2, 0xe6e6bfe6, 0x0e0e380e, 0x1f1f7c1f, - 0x62629562, 0xd4d477d4, 0xa8a89aa8, 0x96966296, - 0xf9f9c3f9, 0xc5c533c5, 0x25259425, 0x59597959, - 0x84842a84, 0x7272d572, 0x3939e439, 0x4c4c2d4c, - 0x5e5e655e, 0x7878fd78, 0x3838e038, 0x8c8c0a8c, - 0xd1d163d1, 0xa5a5aea5, 0xe2e2afe2, 0x61619961, - 0xb3b3f6b3, 0x21218421, 0x9c9c4a9c, 0x1e1e781e, - 0x43431143, 0xc7c73bc7, 0xfcfcd7fc, 0x04041004, - 0x51515951, 0x99995e99, 0x6d6da96d, 0x0d0d340d, - 0xfafacffa, 0xdfdf5bdf, 0x7e7ee57e, 0x24249024, - 0x3b3bec3b, 0xabab96ab, 0xcece1fce, 0x11114411, - 0x8f8f068f, 0x4e4e254e, 0xb7b7e6b7, 0xebeb8beb, - 0x3c3cf03c, 0x81813e81, 0x94946a94, 0xf7f7fbf7, - 0xb9b9deb9, 0x13134c13, 0x2c2cb02c, 0xd3d36bd3, - 0xe7e7bbe7, 0x6e6ea56e, 0xc4c437c4, 0x03030c03, - 0x56564556, 0x44440d44, 0x7f7fe17f, 0xa9a99ea9, - 0x2a2aa82a, 0xbbbbd6bb, 0xc1c123c1, 0x53535153, - 0xdcdc57dc, 0x0b0b2c0b, 0x9d9d4e9d, 0x6c6cad6c, - 0x3131c431, 0x7474cd74, 0xf6f6fff6, 0x46460546, - 0xacac8aac, 0x89891e89, 0x14145014, 0xe1e1a3e1, - 0x16165816, 0x3a3ae83a, 0x6969b969, 0x09092409, - 0x7070dd70, 0xb6b6e2b6, 0xd0d067d0, 0xeded93ed, - 0xcccc17cc, 0x42421542, 0x98985a98, 0xa4a4aaa4, - 0x2828a028, 0x5c5c6d5c, 0xf8f8c7f8, 0x86862286, - }, - { - 0xd8181860, 0x2623238c, 0xb8c6c63f, 0xfbe8e887, - 0xcb878726, 0x11b8b8da, 0x09010104, 0x0d4f4f21, - 0x9b3636d8, 0xffa6a6a2, 0x0cd2d26f, 0x0ef5f5f3, - 0x967979f9, 0x306f6fa1, 0x6d91917e, 0xf8525255, - 0x4760609d, 0x35bcbcca, 0x379b9b56, 0x8a8e8e02, - 0xd2a3a3b6, 0x6c0c0c30, 0x847b7bf1, 0x803535d4, - 0xf51d1d74, 0xb3e0e0a7, 0x21d7d77b, 0x9cc2c22f, - 0x432e2eb8, 0x294b4b31, 0x5dfefedf, 0xd5575741, - 0xbd151554, 0xe87777c1, 0x923737dc, 0x9ee5e5b3, - 0x139f9f46, 0x23f0f0e7, 0x204a4a35, 0x44dada4f, - 0xa258587d, 0xcfc9c903, 0x7c2929a4, 0x5a0a0a28, - 0x50b1b1fe, 0xc9a0a0ba, 0x146b6bb1, 0xd985852e, - 0x3cbdbdce, 0x8f5d5d69, 0x90101040, 0x07f4f4f7, - 0xddcbcb0b, 0xd33e3ef8, 0x2d050514, 0x78676781, - 0x97e4e4b7, 0x0227279c, 0x73414119, 0xa78b8b16, - 0xf6a7a7a6, 0xb27d7de9, 0x4995956e, 0x56d8d847, - 0x70fbfbcb, 0xcdeeee9f, 0xbb7c7ced, 0x71666685, - 0x7bdddd53, 0xaf17175c, 0x45474701, 0x1a9e9e42, - 0xd4caca0f, 0x582d2db4, 0x2ebfbfc6, 0x3f07071c, - 0xacadad8e, 0xb05a5a75, 0xef838336, 0xb63333cc, - 0x5c636391, 0x12020208, 0x93aaaa92, 0xde7171d9, - 0xc6c8c807, 0xd1191964, 0x3b494939, 0x5fd9d943, - 0x31f2f2ef, 0xa8e3e3ab, 0xb95b5b71, 0xbc88881a, - 0x3e9a9a52, 0x0b262698, 0xbf3232c8, 0x59b0b0fa, - 0xf2e9e983, 0x770f0f3c, 0x33d5d573, 0xf480803a, - 0x27bebec2, 0xebcdcd13, 0x893434d0, 0x3248483d, - 0x54ffffdb, 0x8d7a7af5, 0x6490907a, 0x9d5f5f61, - 0x3d202080, 0x0f6868bd, 0xca1a1a68, 0xb7aeae82, - 0x7db4b4ea, 0xce54544d, 0x7f939376, 0x2f222288, - 0x6364648d, 0x2af1f1e3, 0xcc7373d1, 0x82121248, - 0x7a40401d, 0x48080820, 0x95c3c32b, 0xdfecec97, - 0x4ddbdb4b, 0xc0a1a1be, 0x918d8d0e, 0xc83d3df4, - 0x5b979766, 0x00000000, 0xf9cfcf1b, 0x6e2b2bac, - 0xe17676c5, 0xe6828232, 0x28d6d67f, 0xc31b1b6c, - 0x74b5b5ee, 0xbeafaf86, 0x1d6a6ab5, 0xea50505d, - 0x57454509, 0x38f3f3eb, 0xad3030c0, 0xc4efef9b, - 0xda3f3ffc, 0xc7555549, 0xdba2a2b2, 0xe9eaea8f, - 0x6a656589, 0x03babad2, 0x4a2f2fbc, 0x8ec0c027, - 0x60dede5f, 0xfc1c1c70, 0x46fdfdd3, 0x1f4d4d29, - 0x76929272, 0xfa7575c9, 0x36060618, 0xae8a8a12, - 0x4bb2b2f2, 0x85e6e6bf, 0x7e0e0e38, 0xe71f1f7c, - 0x55626295, 0x3ad4d477, 0x81a8a89a, 0x52969662, - 0x62f9f9c3, 0xa3c5c533, 0x10252594, 0xab595979, - 0xd084842a, 0xc57272d5, 0xec3939e4, 0x164c4c2d, - 0x945e5e65, 0x9f7878fd, 0xe53838e0, 0x988c8c0a, - 0x17d1d163, 0xe4a5a5ae, 0xa1e2e2af, 0x4e616199, - 0x42b3b3f6, 0x34212184, 0x089c9c4a, 0xee1e1e78, - 0x61434311, 0xb1c7c73b, 0x4ffcfcd7, 0x24040410, - 0xe3515159, 0x2599995e, 0x226d6da9, 0x650d0d34, - 0x79fafacf, 0x69dfdf5b, 0xa97e7ee5, 0x19242490, - 0xfe3b3bec, 0x9aabab96, 0xf0cece1f, 0x99111144, - 0x838f8f06, 0x044e4e25, 0x66b7b7e6, 0xe0ebeb8b, - 0xc13c3cf0, 0xfd81813e, 0x4094946a, 0x1cf7f7fb, - 0x18b9b9de, 0x8b13134c, 0x512c2cb0, 0x05d3d36b, - 0x8ce7e7bb, 0x396e6ea5, 0xaac4c437, 0x1b03030c, - 0xdc565645, 0x5e44440d, 0xa07f7fe1, 0x88a9a99e, - 0x672a2aa8, 0x0abbbbd6, 0x87c1c123, 0xf1535351, - 0x72dcdc57, 0x530b0b2c, 0x019d9d4e, 0x2b6c6cad, - 0xa43131c4, 0xf37474cd, 0x15f6f6ff, 0x4c464605, - 0xa5acac8a, 0xb589891e, 0xb4141450, 0xbae1e1a3, - 0xa6161658, 0xf73a3ae8, 0x066969b9, 0x41090924, - 0xd77070dd, 0x6fb6b6e2, 0x1ed0d067, 0xd6eded93, - 0xe2cccc17, 0x68424215, 0x2c98985a, 0xeda4a4aa, - 0x752828a0, 0x865c5c6d, 0x6bf8f8c7, 0xc2868622, - }, - { - 0x30d81818, 0x46262323, 0x91b8c6c6, 0xcdfbe8e8, - 0x13cb8787, 0x6d11b8b8, 0x02090101, 0x9e0d4f4f, - 0x6c9b3636, 0x51ffa6a6, 0xb90cd2d2, 0xf70ef5f5, - 0xf2967979, 0xde306f6f, 0x3f6d9191, 0xa4f85252, - 0xc0476060, 0x6535bcbc, 0x2b379b9b, 0x018a8e8e, - 0x5bd2a3a3, 0x186c0c0c, 0xf6847b7b, 0x6a803535, - 0x3af51d1d, 0xddb3e0e0, 0xb321d7d7, 0x999cc2c2, - 0x5c432e2e, 0x96294b4b, 0xe15dfefe, 0xaed55757, - 0x2abd1515, 0xeee87777, 0x6e923737, 0xd79ee5e5, - 0x23139f9f, 0xfd23f0f0, 0x94204a4a, 0xa944dada, - 0xb0a25858, 0x8fcfc9c9, 0x527c2929, 0x145a0a0a, - 0x7f50b1b1, 0x5dc9a0a0, 0xd6146b6b, 0x17d98585, - 0x673cbdbd, 0xba8f5d5d, 0x20901010, 0xf507f4f4, - 0x8bddcbcb, 0x7cd33e3e, 0x0a2d0505, 0xce786767, - 0xd597e4e4, 0x4e022727, 0x82734141, 0x0ba78b8b, - 0x53f6a7a7, 0xfab27d7d, 0x37499595, 0xad56d8d8, - 0xeb70fbfb, 0xc1cdeeee, 0xf8bb7c7c, 0xcc716666, - 0xa77bdddd, 0x2eaf1717, 0x8e454747, 0x211a9e9e, - 0x89d4caca, 0x5a582d2d, 0x632ebfbf, 0x0e3f0707, - 0x47acadad, 0xb4b05a5a, 0x1bef8383, 0x66b63333, - 0xc65c6363, 0x04120202, 0x4993aaaa, 0xe2de7171, - 0x8dc6c8c8, 0x32d11919, 0x923b4949, 0xaf5fd9d9, - 0xf931f2f2, 0xdba8e3e3, 0xb6b95b5b, 0x0dbc8888, - 0x293e9a9a, 0x4c0b2626, 0x64bf3232, 0x7d59b0b0, - 0xcff2e9e9, 0x1e770f0f, 0xb733d5d5, 0x1df48080, - 0x6127bebe, 0x87ebcdcd, 0x68893434, 0x90324848, - 0xe354ffff, 0xf48d7a7a, 0x3d649090, 0xbe9d5f5f, - 0x403d2020, 0xd00f6868, 0x34ca1a1a, 0x41b7aeae, - 0x757db4b4, 0xa8ce5454, 0x3b7f9393, 0x442f2222, - 0xc8636464, 0xff2af1f1, 0xe6cc7373, 0x24821212, - 0x807a4040, 0x10480808, 0x9b95c3c3, 0xc5dfecec, - 0xab4ddbdb, 0x5fc0a1a1, 0x07918d8d, 0x7ac83d3d, - 0x335b9797, 0x00000000, 0x83f9cfcf, 0x566e2b2b, - 0xece17676, 0x19e68282, 0xb128d6d6, 0x36c31b1b, - 0x7774b5b5, 0x43beafaf, 0xd41d6a6a, 0xa0ea5050, - 0x8a574545, 0xfb38f3f3, 0x60ad3030, 0xc3c4efef, - 0x7eda3f3f, 0xaac75555, 0x59dba2a2, 0xc9e9eaea, - 0xca6a6565, 0x6903baba, 0x5e4a2f2f, 0x9d8ec0c0, - 0xa160dede, 0x38fc1c1c, 0xe746fdfd, 0x9a1f4d4d, - 0x39769292, 0xeafa7575, 0x0c360606, 0x09ae8a8a, - 0x794bb2b2, 0xd185e6e6, 0x1c7e0e0e, 0x3ee71f1f, - 0xc4556262, 0xb53ad4d4, 0x4d81a8a8, 0x31529696, - 0xef62f9f9, 0x97a3c5c5, 0x4a102525, 0xb2ab5959, - 0x15d08484, 0xe4c57272, 0x72ec3939, 0x98164c4c, - 0xbc945e5e, 0xf09f7878, 0x70e53838, 0x05988c8c, - 0xbf17d1d1, 0x57e4a5a5, 0xd9a1e2e2, 0xc24e6161, - 0x7b42b3b3, 0x42342121, 0x25089c9c, 0x3cee1e1e, - 0x86614343, 0x93b1c7c7, 0xe54ffcfc, 0x08240404, - 0xa2e35151, 0x2f259999, 0xda226d6d, 0x1a650d0d, - 0xe979fafa, 0xa369dfdf, 0xfca97e7e, 0x48192424, - 0x76fe3b3b, 0x4b9aabab, 0x81f0cece, 0x22991111, - 0x03838f8f, 0x9c044e4e, 0x7366b7b7, 0xcbe0ebeb, - 0x78c13c3c, 0x1ffd8181, 0x35409494, 0xf31cf7f7, - 0x6f18b9b9, 0x268b1313, 0x58512c2c, 0xbb05d3d3, - 0xd38ce7e7, 0xdc396e6e, 0x95aac4c4, 0x061b0303, - 0xacdc5656, 0x885e4444, 0xfea07f7f, 0x4f88a9a9, - 0x54672a2a, 0x6b0abbbb, 0x9f87c1c1, 0xa6f15353, - 0xa572dcdc, 0x16530b0b, 0x27019d9d, 0xd82b6c6c, - 0x62a43131, 0xe8f37474, 0xf115f6f6, 0x8c4c4646, - 0x45a5acac, 0x0fb58989, 0x28b41414, 0xdfbae1e1, - 0x2ca61616, 0x74f73a3a, 0xd2066969, 0x12410909, - 0xe0d77070, 0x716fb6b6, 0xbd1ed0d0, 0xc7d6eded, - 0x85e2cccc, 0x84684242, 0x2d2c9898, 0x55eda4a4, - 0x50752828, 0xb8865c5c, 0xed6bf8f8, 0x11c28686, - }, - { - 0x7830d818, 0xaf462623, 0xf991b8c6, 0x6fcdfbe8, - 0xa113cb87, 0x626d11b8, 0x05020901, 0x6e9e0d4f, - 0xee6c9b36, 0x0451ffa6, 0xbdb90cd2, 0x06f70ef5, - 0x80f29679, 0xcede306f, 0xef3f6d91, 0x07a4f852, - 0xfdc04760, 0x766535bc, 0xcd2b379b, 0x8c018a8e, - 0x155bd2a3, 0x3c186c0c, 0x8af6847b, 0xe16a8035, - 0x693af51d, 0x47ddb3e0, 0xacb321d7, 0xed999cc2, - 0x965c432e, 0x7a96294b, 0x21e15dfe, 0x16aed557, - 0x412abd15, 0xb6eee877, 0xeb6e9237, 0x56d79ee5, - 0xd923139f, 0x17fd23f0, 0x7f94204a, 0x95a944da, - 0x25b0a258, 0xca8fcfc9, 0x8d527c29, 0x22145a0a, - 0x4f7f50b1, 0x1a5dc9a0, 0xdad6146b, 0xab17d985, - 0x73673cbd, 0x34ba8f5d, 0x50209010, 0x03f507f4, - 0xc08bddcb, 0xc67cd33e, 0x110a2d05, 0xe6ce7867, - 0x53d597e4, 0xbb4e0227, 0x58827341, 0x9d0ba78b, - 0x0153f6a7, 0x94fab27d, 0xfb374995, 0x9fad56d8, - 0x30eb70fb, 0x71c1cdee, 0x91f8bb7c, 0xe3cc7166, - 0x8ea77bdd, 0x4b2eaf17, 0x468e4547, 0xdc211a9e, - 0xc589d4ca, 0x995a582d, 0x79632ebf, 0x1b0e3f07, - 0x2347acad, 0x2fb4b05a, 0xb51bef83, 0xff66b633, - 0xf2c65c63, 0x0a041202, 0x384993aa, 0xa8e2de71, - 0xcf8dc6c8, 0x7d32d119, 0x70923b49, 0x9aaf5fd9, - 0x1df931f2, 0x48dba8e3, 0x2ab6b95b, 0x920dbc88, - 0xc8293e9a, 0xbe4c0b26, 0xfa64bf32, 0x4a7d59b0, - 0x6acff2e9, 0x331e770f, 0xa6b733d5, 0xba1df480, - 0x7c6127be, 0xde87ebcd, 0xe4688934, 0x75903248, - 0x24e354ff, 0x8ff48d7a, 0xea3d6490, 0x3ebe9d5f, - 0xa0403d20, 0xd5d00f68, 0x7234ca1a, 0x2c41b7ae, - 0x5e757db4, 0x19a8ce54, 0xe53b7f93, 0xaa442f22, - 0xe9c86364, 0x12ff2af1, 0xa2e6cc73, 0x5a248212, - 0x5d807a40, 0x28104808, 0xe89b95c3, 0x7bc5dfec, - 0x90ab4ddb, 0x1f5fc0a1, 0x8307918d, 0xc97ac83d, - 0xf1335b97, 0x00000000, 0xd483f9cf, 0x87566e2b, - 0xb3ece176, 0xb019e682, 0xa9b128d6, 0x7736c31b, - 0x5b7774b5, 0x2943beaf, 0xdfd41d6a, 0x0da0ea50, - 0x4c8a5745, 0x18fb38f3, 0xf060ad30, 0x74c3c4ef, - 0xc37eda3f, 0x1caac755, 0x1059dba2, 0x65c9e9ea, - 0xecca6a65, 0x686903ba, 0x935e4a2f, 0xe79d8ec0, - 0x81a160de, 0x6c38fc1c, 0x2ee746fd, 0x649a1f4d, - 0xe0397692, 0xbceafa75, 0x1e0c3606, 0x9809ae8a, - 0x40794bb2, 0x59d185e6, 0x361c7e0e, 0x633ee71f, - 0xf7c45562, 0xa3b53ad4, 0x324d81a8, 0xf4315296, - 0x3aef62f9, 0xf697a3c5, 0xb14a1025, 0x20b2ab59, - 0xae15d084, 0xa7e4c572, 0xdd72ec39, 0x6198164c, - 0x3bbc945e, 0x85f09f78, 0xd870e538, 0x8605988c, - 0xb2bf17d1, 0x0b57e4a5, 0x4dd9a1e2, 0xf8c24e61, - 0x457b42b3, 0xa5423421, 0xd625089c, 0x663cee1e, - 0x52866143, 0xfc93b1c7, 0x2be54ffc, 0x14082404, - 0x08a2e351, 0xc72f2599, 0xc4da226d, 0x391a650d, - 0x35e979fa, 0x84a369df, 0x9bfca97e, 0xb4481924, - 0xd776fe3b, 0x3d4b9aab, 0xd181f0ce, 0x55229911, - 0x8903838f, 0x6b9c044e, 0x517366b7, 0x60cbe0eb, - 0xcc78c13c, 0xbf1ffd81, 0xfe354094, 0x0cf31cf7, - 0x676f18b9, 0x5f268b13, 0x9c58512c, 0xb8bb05d3, - 0x5cd38ce7, 0xcbdc396e, 0xf395aac4, 0x0f061b03, - 0x13acdc56, 0x49885e44, 0x9efea07f, 0x374f88a9, - 0x8254672a, 0x6d6b0abb, 0xe29f87c1, 0x02a6f153, - 0x8ba572dc, 0x2716530b, 0xd327019d, 0xc1d82b6c, - 0xf562a431, 0xb9e8f374, 0x09f115f6, 0x438c4c46, - 0x2645a5ac, 0x970fb589, 0x4428b414, 0x42dfbae1, - 0x4e2ca616, 0xd274f73a, 0xd0d20669, 0x2d124109, - 0xade0d770, 0x54716fb6, 0xb7bd1ed0, 0x7ec7d6ed, - 0xdb85e2cc, 0x57846842, 0xc22d2c98, 0x0e55eda4, - 0x88507528, 0x31b8865c, 0x3fed6bf8, 0xa411c286, - }, -}; - -__constant u32a rch[R + 1] = -{ - 0x00000000, - 0x1823c6e8, - 0x36a6d2f5, - 0x60bc9b8e, - 0x1de0d7c2, - 0x157737e5, - 0x58c9290a, - 0xbd5d10f4, - 0xe427418b, - 0xfbee7c66, - 0xca2dbf07, -}; - -__constant u32a rcl[R + 1] = -{ - 0x00000000, - 0x87b8014f, - 0x796f9152, - 0xa30c7b35, - 0x2e4bfe57, - 0x9ff04ada, - 0xb1a06b85, - 0xcb3e0567, - 0xa77d95d8, - 0xdd17479e, - 0xad5a8333, -}; - -// this is a highly optimized that assumes dgst[16] = { 0 }; only reuse of no 2nd transform is needed - -void whirlpool_transform (const u32x w[16], u32x dgst[16], __local u32 (*s_Ch)[256], __local u32 (*s_Cl)[256]) -{ - u32x Kh[8]; - u32x Kl[8]; - - Kh[0] = 0x300beec0; - Kl[0] = 0xaf902967; - Kh[1] = 0x28282828; - Kl[1] = 0x28282828; - Kh[2] = 0x28282828; - Kl[2] = 0x28282828; - Kh[3] = 0x28282828; - Kl[3] = 0x28282828; - Kh[4] = 0x28282828; - Kl[4] = 0x28282828; - Kh[5] = 0x28282828; - Kl[5] = 0x28282828; - Kh[6] = 0x28282828; - Kl[6] = 0x28282828; - Kh[7] = 0x28282828; - Kl[7] = 0x28282828; - - u32x stateh[8]; - u32x statel[8]; - - stateh[0] = w[ 0]; - statel[0] = w[ 1]; - stateh[1] = w[ 2]; - statel[1] = w[ 3]; - stateh[2] = w[ 4]; - statel[2] = w[ 5]; - stateh[3] = w[ 6]; - statel[3] = w[ 7]; - stateh[4] = w[ 8]; - statel[4] = w[ 9]; - stateh[5] = w[10]; - statel[5] = w[11]; - stateh[6] = w[12]; - statel[6] = w[13]; - stateh[7] = w[14]; - statel[7] = w[15]; - - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - - for (int r = 2; r <= R; r++) - { - u32x Lh[8]; - u32x Ll[8]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = Kh[(i + 8) & 7] >> 24; - const u32x Lp1 = Kh[(i + 7) & 7] >> 16; - const u32x Lp2 = Kh[(i + 6) & 7] >> 8; - const u32x Lp3 = Kh[(i + 5) & 7] >> 0; - const u32x Lp4 = Kl[(i + 4) & 7] >> 24; - const u32x Lp5 = Kl[(i + 3) & 7] >> 16; - const u32x Lp6 = Kl[(i + 2) & 7] >> 8; - const u32x Lp7 = Kl[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - Kh[0] = Lh[0] ^ rch[r]; - Kl[0] = Ll[0] ^ rcl[r]; - Kh[1] = Lh[1]; - Kl[1] = Ll[1]; - Kh[2] = Lh[2]; - Kl[2] = Ll[2]; - Kh[3] = Lh[3]; - Kl[3] = Ll[3]; - Kh[4] = Lh[4]; - Kl[4] = Ll[4]; - Kh[5] = Lh[5]; - Kl[5] = Ll[5]; - Kh[6] = Lh[6]; - Kl[6] = Ll[6]; - Kh[7] = Lh[7]; - Kl[7] = Ll[7]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) - { - const u32x Lp0 = stateh[(i + 8) & 7] >> 24; - const u32x Lp1 = stateh[(i + 7) & 7] >> 16; - const u32x Lp2 = stateh[(i + 6) & 7] >> 8; - const u32x Lp3 = stateh[(i + 5) & 7] >> 0; - const u32x Lp4 = statel[(i + 4) & 7] >> 24; - const u32x Lp5 = statel[(i + 3) & 7] >> 16; - const u32x Lp6 = statel[(i + 2) & 7] >> 8; - const u32x Lp7 = statel[(i + 1) & 7] >> 0; - - Lh[i] = BOX (s_Ch, 0, Lp0 & 0xff) - ^ BOX (s_Ch, 1, Lp1 & 0xff) - ^ BOX (s_Ch, 2, Lp2 & 0xff) - ^ BOX (s_Ch, 3, Lp3 & 0xff) - ^ BOX (s_Ch, 4, Lp4 & 0xff) - ^ BOX (s_Ch, 5, Lp5 & 0xff) - ^ BOX (s_Ch, 6, Lp6 & 0xff) - ^ BOX (s_Ch, 7, Lp7 & 0xff); - - Ll[i] = BOX (s_Cl, 0, Lp0 & 0xff) - ^ BOX (s_Cl, 1, Lp1 & 0xff) - ^ BOX (s_Cl, 2, Lp2 & 0xff) - ^ BOX (s_Cl, 3, Lp3 & 0xff) - ^ BOX (s_Cl, 4, Lp4 & 0xff) - ^ BOX (s_Cl, 5, Lp5 & 0xff) - ^ BOX (s_Cl, 6, Lp6 & 0xff) - ^ BOX (s_Cl, 7, Lp7 & 0xff); - } - - stateh[0] = Lh[0] ^ Kh[0]; - statel[0] = Ll[0] ^ Kl[0]; - stateh[1] = Lh[1] ^ Kh[1]; - statel[1] = Ll[1] ^ Kl[1]; - stateh[2] = Lh[2] ^ Kh[2]; - statel[2] = Ll[2] ^ Kl[2]; - stateh[3] = Lh[3] ^ Kh[3]; - statel[3] = Ll[3] ^ Kl[3]; - stateh[4] = Lh[4] ^ Kh[4]; - statel[4] = Ll[4] ^ Kl[4]; - stateh[5] = Lh[5] ^ Kh[5]; - statel[5] = Ll[5] ^ Kl[5]; - stateh[6] = Lh[6] ^ Kh[6]; - statel[6] = Ll[6] ^ Kl[6]; - stateh[7] = Lh[7] ^ Kh[7]; - statel[7] = Ll[7] ^ Kl[7]; - } - - dgst[ 0] = stateh[0] ^ w[ 0]; - dgst[ 1] = statel[0] ^ w[ 1]; - dgst[ 2] = stateh[1] ^ w[ 2]; - dgst[ 3] = statel[1] ^ w[ 3]; - dgst[ 4] = stateh[2] ^ w[ 4]; - dgst[ 5] = statel[2] ^ w[ 5]; - dgst[ 6] = stateh[3] ^ w[ 6]; - dgst[ 7] = statel[3] ^ w[ 7]; - dgst[ 8] = stateh[4] ^ w[ 8]; - dgst[ 9] = statel[4] ^ w[ 9]; - dgst[10] = stateh[5] ^ w[10]; - dgst[11] = statel[5] ^ w[11]; - dgst[12] = stateh[6] ^ w[12]; - dgst[13] = statel[6] ^ w[13]; - dgst[14] = stateh[7] ^ w[14]; - dgst[15] = statel[7] ^ w[15]; -} - -void m06100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 (*s_Cl)[256], __local u32 (*s_Ch)[256]) +__kernel void m06100_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1349,68 +21,89 @@ void m06100m (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w[16]; + w[0] = w0; - w[ 0] = w0lr; - w[ 1] = w0[1]; - w[ 2] = w0[2]; - w[ 3] = w0[3]; - w[ 4] = w1[0]; - w[ 5] = w1[1]; - w[ 6] = w1[2]; - w[ 7] = w1[3]; - w[ 8] = w2[0]; - w[ 9] = w2[1]; - w[10] = w2[2]; - w[11] = w2[3]; - w[12] = w3[0]; - w[13] = w3[1]; - w[14] = 0; - w[15] = pw_len * 8; + whirlpool_ctx_vector_t ctx; - /** - * Whirlool - */ + whirlpool_init_vector (&ctx, s_Ch, s_Cl); - u32x dgst[16]; + whirlpool_update_vector (&ctx, w, pw_len); - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; + whirlpool_final_vector (&ctx); - whirlpool_transform (w, dgst, s_Ch, s_Cl); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - COMPARE_M_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m06100s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, __local u32 (*s_Cl)[256], __local u32 (*s_Ch)[256]) +__kernel void m06100_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier @@ -1418,6 +111,39 @@ void m06100s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * shared + */ + + __local u32 s_Ch[8][256]; + __local u32 s_Cl[8][256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_Ch[0][i] = Ch[0][i]; + s_Ch[1][i] = Ch[1][i]; + s_Ch[2][i] = Ch[2][i]; + s_Ch[3][i] = Ch[3][i]; + s_Ch[4][i] = Ch[4][i]; + s_Ch[5][i] = Ch[5][i]; + s_Ch[6][i] = Ch[6][i]; + s_Ch[7][i] = Ch[7][i]; + + s_Cl[0][i] = Cl[0][i]; + s_Cl[1][i] = Cl[1][i]; + s_Cl[2][i] = Cl[2][i]; + s_Cl[3][i] = Cl[3][i]; + s_Cl[4][i] = Cl[4][i]; + s_Cl[5][i] = Cl[5][i]; + s_Cl[6][i] = Cl[6][i]; + s_Cl[7][i] = Cl[7][i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + if (gid >= gid_max) return; /** * digest @@ -1431,560 +157,50 @@ void m06100s (u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const u32 pw_len, __gl digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w0[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { - const u32x w0r = ix_create_bft (bfs_buf, il_pos); + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - const u32x w0lr = w0l | w0r; + const u32x w0 = w0l | w0r; - u32x w[16]; + w[0] = w0; - w[ 0] = w0lr; - w[ 1] = w0[1]; - w[ 2] = w0[2]; - w[ 3] = w0[3]; - w[ 4] = w1[0]; - w[ 5] = w1[1]; - w[ 6] = w1[2]; - w[ 7] = w1[3]; - w[ 8] = w2[0]; - w[ 9] = w2[1]; - w[10] = w2[2]; - w[11] = w2[3]; - w[12] = w3[0]; - w[13] = w3[1]; - w[14] = 0; - w[15] = pw_len * 8; + whirlpool_ctx_vector_t ctx; - /** - * Whirlool - */ + whirlpool_init_vector (&ctx, s_Ch, s_Cl); - u32x dgst[16]; + whirlpool_update_vector (&ctx, w, pw_len); - dgst[ 0] = 0; - dgst[ 1] = 0; - dgst[ 2] = 0; - dgst[ 3] = 0; - dgst[ 4] = 0; - dgst[ 5] = 0; - dgst[ 6] = 0; - dgst[ 7] = 0; - dgst[ 8] = 0; - dgst[ 9] = 0; - dgst[10] = 0; - dgst[11] = 0; - dgst[12] = 0; - dgst[13] = 0; - dgst[14] = 0; - dgst[15] = 0; + whirlpool_final_vector (&ctx); - whirlpool_transform (w, dgst, s_Ch, s_Cl); + const u32x r0 = ctx.h[DGST_R0]; + const u32x r1 = ctx.h[DGST_R1]; + const u32x r2 = ctx.h[DGST_R2]; + const u32x r3 = ctx.h[DGST_R3]; - COMPARE_S_SIMD (dgst[0], dgst[1], dgst[2], dgst[3]); + COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m06100_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} - -__kernel void m06100_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} - -__kernel void m06100_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100m (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} - -__kernel void m06100_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = 0; - w1[1] = 0; - w1[2] = 0; - w1[3] = 0; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} - -__kernel void m06100_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - - u32 w3[4]; - - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} - -__kernel void m06100_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * shared - */ - - __local u32 s_Ch[8][256]; - __local u32 s_Cl[8][256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_Ch[0][i] = Ch[0][i]; - s_Ch[1][i] = Ch[1][i]; - s_Ch[2][i] = Ch[2][i]; - s_Ch[3][i] = Ch[3][i]; - s_Ch[4][i] = Ch[4][i]; - s_Ch[5][i] = Ch[5][i]; - s_Ch[6][i] = Ch[6][i]; - s_Ch[7][i] = Ch[7][i]; - - s_Cl[0][i] = Cl[0][i]; - s_Cl[1][i] = Cl[1][i]; - s_Cl[2][i] = Cl[2][i]; - s_Cl[3][i] = Cl[3][i]; - s_Cl[4][i] = Cl[4][i]; - s_Cl[5][i] = Cl[5][i]; - s_Cl[6][i] = Cl[6][i]; - s_Cl[7][i] = Cl[7][i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - if (gid >= gid_max) return; - - /** - * base - */ - - u32 w0[4]; - - w0[0] = pws[gid].i[ 0]; - w0[1] = pws[gid].i[ 1]; - w0[2] = pws[gid].i[ 2]; - w0[3] = pws[gid].i[ 3]; - - u32 w1[4]; - - w1[0] = pws[gid].i[ 4]; - w1[1] = pws[gid].i[ 5]; - w1[2] = pws[gid].i[ 6]; - w1[3] = pws[gid].i[ 7]; - - u32 w2[4]; - - w2[0] = pws[gid].i[ 8]; - w2[1] = pws[gid].i[ 9]; - w2[2] = pws[gid].i[10]; - w2[3] = pws[gid].i[11]; - - u32 w3[4]; - - w3[0] = pws[gid].i[12]; - w3[1] = pws[gid].i[13]; - w3[2] = 0; - w3[3] = 0; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m06100s (w0, w1, w2, w3, pw_len, pws, rules_buf, combs_buf, bfs_buf, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset, s_Cl, s_Ch); -} diff --git a/OpenCL/m06300-optimized.cl b/OpenCL/m06300-optimized.cl new file mode 100644 index 000000000..0645a12fd --- /dev/null +++ b/OpenCL/m06300-optimized.cl @@ -0,0 +1,1033 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + + u32 w0_t = w0[0]; + u32 w1_t = w0[1]; + u32 w2_t = w0[2]; + u32 w3_t = w0[3]; + u32 w4_t = w1[0]; + u32 w5_t = w1[1]; + u32 w6_t = w1[2]; + u32 w7_t = w1[3]; + u32 w8_t = w2[0]; + u32 w9_t = w2[1]; + u32 wa_t = w2[2]; + u32 wb_t = w2[3]; + u32 wc_t = w3[0]; + u32 wd_t = w3[1]; + u32 we_t = w3[2]; + u32 wf_t = 0; + + MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); + MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); + MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); + MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); + MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); + + MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); + MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); + MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); + MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); + MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); + + MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); + MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); + MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); + MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); + MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); + + MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); + MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); + MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); + MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); + MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; +} + +void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0x80; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0x80, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + block0[3] = tmp3; + block1[0] = tmp4; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + block1[0] = tmp3; + block1[1] = tmp4; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + block1[1] = tmp3; + block1[2] = tmp4; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + block1[2] = tmp3; + block1[3] = tmp4; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + block1[3] = tmp3; + block2[0] = tmp4; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + block2[0] = tmp3; + block2[1] = tmp4; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + block2[1] = tmp3; + block2[2] = tmp4; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + block2[2] = tmp3; + block2[3] = tmp4; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + block2[3] = tmp3; + block3[0] = tmp4; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + block3[0] = tmp3; + block3[1] = tmp4; + break; + } +} + +void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) +{ + u32 tmp0; + u32 tmp1; + u32 tmp2; + + #if defined IS_AMD || defined IS_GENERIC + + const int offset_minus_4 = 4 - (block_len & 3); + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); + + const u32 mod = block_len & 3; + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = 0; + } + + #endif + + #ifdef IS_NV + + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], 0, selector); + + #endif + + const u32 div = block_len / 4; + + switch (div) + { + case 0: block0[0] |= tmp0; + block0[1] = tmp1; + block0[2] = tmp2; + break; + case 1: block0[1] |= tmp0; + block0[2] = tmp1; + block0[3] = tmp2; + break; + case 2: block0[2] |= tmp0; + block0[3] = tmp1; + block1[0] = tmp2; + break; + case 3: block0[3] |= tmp0; + block1[0] = tmp1; + block1[1] = tmp2; + break; + case 4: block1[0] |= tmp0; + block1[1] = tmp1; + block1[2] = tmp2; + break; + case 5: block1[1] |= tmp0; + block1[2] = tmp1; + block1[3] = tmp2; + break; + case 6: block1[2] |= tmp0; + block1[3] = tmp1; + block2[0] = tmp2; + break; + case 7: block1[3] |= tmp0; + block2[0] = tmp1; + block2[1] = tmp2; + break; + case 8: block2[0] |= tmp0; + block2[1] = tmp1; + block2[2] = tmp2; + break; + case 9: block2[1] |= tmp0; + block2[2] = tmp1; + block2[3] = tmp2; + break; + case 10: block2[2] |= tmp0; + block2[3] = tmp1; + block3[0] = tmp2; + break; + case 11: block2[3] |= tmp0; + block3[0] = tmp1; + block3[1] = tmp2; + break; + } +} + +void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) +{ + switch (block_len) + { + case 0: + block0[0] = append; + break; + + case 1: + block0[0] = block0[0] | append << 8; + break; + + case 2: + block0[0] = block0[0] | append << 16; + break; + + case 3: + block0[0] = block0[0] | append << 24; + break; + + case 4: + block0[1] = append; + break; + + case 5: + block0[1] = block0[1] | append << 8; + break; + + case 6: + block0[1] = block0[1] | append << 16; + break; + + case 7: + block0[1] = block0[1] | append << 24; + break; + + case 8: + block0[2] = append; + break; + + case 9: + block0[2] = block0[2] | append << 8; + break; + + case 10: + block0[2] = block0[2] | append << 16; + break; + + case 11: + block0[2] = block0[2] | append << 24; + break; + + case 12: + block0[3] = append; + break; + + case 13: + block0[3] = block0[3] | append << 8; + break; + + case 14: + block0[3] = block0[3] | append << 16; + break; + + case 15: + block0[3] = block0[3] | append << 24; + break; + + case 16: + block1[0] = append; + break; + + case 17: + block1[0] = block1[0] | append << 8; + break; + + case 18: + block1[0] = block1[0] | append << 16; + break; + + case 19: + block1[0] = block1[0] | append << 24; + break; + + case 20: + block1[1] = append; + break; + + case 21: + block1[1] = block1[1] | append << 8; + break; + + case 22: + block1[1] = block1[1] | append << 16; + break; + + case 23: + block1[1] = block1[1] | append << 24; + break; + + case 24: + block1[2] = append; + break; + + case 25: + block1[2] = block1[2] | append << 8; + break; + + case 26: + block1[2] = block1[2] | append << 16; + break; + + case 27: + block1[2] = block1[2] | append << 24; + break; + + case 28: + block1[3] = append; + break; + + case 29: + block1[3] = block1[3] | append << 8; + break; + + case 30: + block1[3] = block1[3] | append << 16; + break; + + case 31: + block1[3] = block1[3] | append << 24; + break; + + case 32: + block2[0] = append; + break; + + case 33: + block2[0] = block2[0] | append << 8; + break; + + case 34: + block2[0] = block2[0] | append << 16; + break; + + case 35: + block2[0] = block2[0] | append << 24; + break; + + case 36: + block2[1] = append; + break; + + case 37: + block2[1] = block2[1] | append << 8; + break; + + case 38: + block2[1] = block2[1] | append << 16; + break; + + case 39: + block2[1] = block2[1] | append << 24; + break; + + case 40: + block2[2] = append; + break; + + case 41: + block2[2] = block2[2] | append << 8; + break; + + case 42: + block2[2] = block2[2] | append << 16; + break; + + case 43: + block2[2] = block2[2] | append << 24; + break; + + case 44: + block2[3] = append; + break; + + case 45: + block2[3] = block2[3] | append << 8; + break; + + case 46: + block2[3] = block2[3] | append << 16; + break; + + case 47: + block2[3] = block2[3] | append << 24; + break; + + case 48: + block3[0] = append; + break; + + case 49: + block3[0] = block3[0] | append << 8; + break; + + case 50: + block3[0] = block3[0] | append << 16; + break; + + case 51: + block3[0] = block3[0] | append << 24; + break; + + case 52: + block3[1] = append; + break; + + case 53: + block3[1] = block3[1] | append << 8; + break; + + case 54: + block3[1] = block3[1] | append << 16; + break; + + case 55: + block3[1] = block3[1] | append << 24; + break; + + case 56: + block3[2] = append; + break; + } +} + +__kernel void m06300_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * init + */ + + //memcat16 (block0, block1, block2, block3, block_len, w0); + //block_len += pw_len; + + u32 block_len = pw_len; + + u32 block0[4]; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + u32 digest[4]; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + /* The password first, since that is what is most unknown */ + /* Then the raw salt */ + /* Then just as many characters of the MD5(pw,salt,pw) */ + + //memcat16 (block0, block1, block2, block3, block_len, w); + //block_len += pw_len; + + block_len = pw_len; + + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + + truncate_block_4x4_le (digest, pw_len); + + memcat16 (block0, block1, block2, block3, block_len, digest); + + block_len += pw_len; + + /* Then something really weird... */ + + u32 append = block0[0] & 0xFF; + + for (u32 j = pw_len; j; j >>= 1) + { + if ((j & 1) == 0) + { + append_1st (block0, block1, block2, block3, block_len, append); + } + + block_len++; + } + + append_0x80_4x4 (block0, block1, block2, block3, block_len); + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m06300_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + u32 w0_x80[4]; + + w0_x80[0] = w0[0]; + w0_x80[1] = w0[1]; + w0_x80[2] = w0[2]; + w0_x80[3] = w0[3]; + + append_0x80_1x4 (w0_x80, pw_len); + + /** + * salt + */ + + u32 salt_buf[2]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * digest + */ + + u32 digest[4]; + + digest[0] = tmps[gid].digest_buf[0]; + digest[1] = tmps[gid].digest_buf[1]; + digest[2] = tmps[gid].digest_buf[2]; + digest[3] = tmps[gid].digest_buf[3]; + + /** + * loop + */ + + /* and now, just to make sure things don't run too fast */ + + u32 block_len; + + u32 block0[4]; + + block0[0] = 0; + block0[1] = 0; + block0[2] = 0; + block0[3] = 0; + + u32 block1[4]; + + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + + u32 block2[4]; + + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + + u32 block3[4]; + + block3[0] = 0; + block3[1] = 0; + block3[2] = 0; + block3[3] = 0; + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + block1[0] = 0; + block1[1] = 0; + block1[2] = 0; + block1[3] = 0; + block2[0] = 0; + block2[1] = 0; + block2[2] = 0; + block2[3] = 0; + block3[0] = 0; + block3[1] = 0; + + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 1 : 0; + const u32 j7 = (j % 7) ? 1 : 0; + + if (j1) + { + block0[0] = w0[0]; + block0[1] = w0[1]; + block0[2] = w0[2]; + block0[3] = w0[3]; + + block_len = pw_len; + + if (j3) + { + memcat8 (block0, block1, block2, block3, block_len, salt_buf); + + block_len += salt_len; + } + + if (j7) + { + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + + memcat16_x80 (block0, block1, block2, block3, block_len, digest); + + block_len += 16; + } + else + { + block0[0] = digest[0]; + block0[1] = digest[1]; + block0[2] = digest[2]; + block0[3] = digest[3]; + + block_len = 16; + + if (j3 && j7) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + + memcat16 (block0, block1, block2, block3, block_len, w0); + + block_len += pw_len; + } + else if (j3) + { + block1[0] = salt_buf[0]; + block1[1] = salt_buf[1]; + + block_len += salt_len; + } + else if (j7) + { + block1[0] = w0[0]; + block1[1] = w0[1]; + block1[2] = w0[2]; + block1[3] = w0[3]; + + block_len += pw_len; + } + + memcat16 (block0, block1, block2, block3, block_len, w0_x80); + + block_len += pw_len; + } + + block3[2] = block_len * 8; + + digest[0] = MD5M_A; + digest[1] = MD5M_B; + digest[2] = MD5M_C; + digest[3] = MD5M_D; + + md5_transform (block0, block1, block2, block3, digest); + } + + tmps[gid].digest_buf[0] = digest[0]; + tmps[gid].digest_buf[1] = digest[1]; + tmps[gid].digest_buf[2] = digest[2]; + tmps[gid].digest_buf[3] = digest[3]; +} + +__kernel void m06300_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 r0 = tmps[gid].digest_buf[DGST_R0]; + const u32 r1 = tmps[gid].digest_buf[DGST_R1]; + const u32 r2 = tmps[gid].digest_buf[DGST_R2]; + const u32 r3 = tmps[gid].digest_buf[DGST_R3]; + + #define il_pos 0 + + #include COMPARE_M +} diff --git a/OpenCL/m06300-pure.cl b/OpenCL/m06300-pure.cl deleted file mode 100644 index dbe156e8d..000000000 --- a/OpenCL/m06300-pure.cl +++ /dev/null @@ -1,253 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_md5.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -__kernel void m06300_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * prepare - */ - - md5_ctx_t md5_ctx1; - - md5_init (&md5_ctx1); - - md5_update (&md5_ctx1, w, pw_len); - - md5_update (&md5_ctx1, s, salt_len); - - md5_update (&md5_ctx1, w, pw_len); - - md5_final (&md5_ctx1); - - u32 final[16] = { 0 }; - - final[0] = md5_ctx1.h[0]; - final[1] = md5_ctx1.h[1]; - final[2] = md5_ctx1.h[2]; - final[3] = md5_ctx1.h[3]; - - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - md5_update (&md5_ctx, w, pw_len); - - md5_update (&md5_ctx, s, salt_len); - - int pl; - - for (pl = pw_len; pl > 16; pl -= 16) - { - md5_update (&md5_ctx, final, 16); - } - - truncate_block_4x4_le (final, pl); - - md5_update (&md5_ctx, final, pl); - - /* Then something really weird... */ - - for (int i = pw_len; i != 0; i >>= 1) - { - u32 t[16] = { 0 }; - - if (i & 1) - { - t[0] = 0; - } - else - { - t[0] = w[0] & 0xff; - } - - md5_update (&md5_ctx, t, 1); - } - - md5_final (&md5_ctx); - - tmps[gid].digest_buf[0] = md5_ctx.h[0]; - tmps[gid].digest_buf[1] = md5_ctx.h[1]; - tmps[gid].digest_buf[2] = md5_ctx.h[2]; - tmps[gid].digest_buf[3] = md5_ctx.h[3]; -} - -__kernel void m06300_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * digest - */ - - u32 digest[16] = { 0 }; - - digest[0] = tmps[gid].digest_buf[0]; - digest[1] = tmps[gid].digest_buf[1]; - digest[2] = tmps[gid].digest_buf[2]; - digest[3] = tmps[gid].digest_buf[3]; - - /** - * loop - */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - md5_ctx_t md5_ctx; - - md5_init (&md5_ctx); - - if (j & 1) - { - md5_update (&md5_ctx, w, pw_len); - } - else - { - md5_update (&md5_ctx, digest, 16); - } - - if (j % 3) - { - md5_update (&md5_ctx, s, salt_len); - } - - if (j % 7) - { - md5_update (&md5_ctx, w, pw_len); - } - - if (j & 1) - { - md5_update (&md5_ctx, digest, 16); - } - else - { - md5_update (&md5_ctx, w, pw_len); - } - - md5_final (&md5_ctx); - - digest[0] = md5_ctx.h[0]; - digest[1] = md5_ctx.h[1]; - digest[2] = md5_ctx.h[2]; - digest[3] = md5_ctx.h[3]; - } - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; -} - -__kernel void m06300_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - /** - * digest - */ - - const u32 r0 = tmps[gid].digest_buf[DGST_R0]; - const u32 r1 = tmps[gid].digest_buf[DGST_R1]; - const u32 r2 = tmps[gid].digest_buf[DGST_R2]; - const u32 r3 = tmps[gid].digest_buf[DGST_R3]; - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m06300.cl b/OpenCL/m06300.cl index 0645a12fd..dbe156e8d 100644 --- a/OpenCL/m06300.cl +++ b/OpenCL/m06300.cl @@ -3,663 +3,18 @@ * License.....: MIT */ +//#define NEW_SIMD_CODE + #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_md5.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" -void md5_transform (const u32 w0[4], const u32 w1[4], const u32 w2[4], const u32 w3[4], u32 digest[4]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - - u32 w0_t = w0[0]; - u32 w1_t = w0[1]; - u32 w2_t = w0[2]; - u32 w3_t = w0[3]; - u32 w4_t = w1[0]; - u32 w5_t = w1[1]; - u32 w6_t = w1[2]; - u32 w7_t = w1[3]; - u32 w8_t = w2[0]; - u32 w9_t = w2[1]; - u32 wa_t = w2[2]; - u32 wb_t = w2[3]; - u32 wc_t = w3[0]; - u32 wd_t = w3[1]; - u32 we_t = w3[2]; - u32 wf_t = 0; - - MD5_STEP (MD5_Fo, a, b, c, d, w0_t, MD5C00, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w1_t, MD5C01, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w2_t, MD5C02, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w3_t, MD5C03, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w4_t, MD5C04, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w5_t, MD5C05, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, w6_t, MD5C06, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, w7_t, MD5C07, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, w8_t, MD5C08, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, w9_t, MD5C09, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, wa_t, MD5C0a, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wb_t, MD5C0b, MD5S03); - MD5_STEP (MD5_Fo, a, b, c, d, wc_t, MD5C0c, MD5S00); - MD5_STEP (MD5_Fo, d, a, b, c, wd_t, MD5C0d, MD5S01); - MD5_STEP (MD5_Fo, c, d, a, b, we_t, MD5C0e, MD5S02); - MD5_STEP (MD5_Fo, b, c, d, a, wf_t, MD5C0f, MD5S03); - - MD5_STEP (MD5_Go, a, b, c, d, w1_t, MD5C10, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w6_t, MD5C11, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wb_t, MD5C12, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w0_t, MD5C13, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w5_t, MD5C14, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, wa_t, MD5C15, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, wf_t, MD5C16, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w4_t, MD5C17, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, w9_t, MD5C18, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, we_t, MD5C19, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w3_t, MD5C1a, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, w8_t, MD5C1b, MD5S13); - MD5_STEP (MD5_Go, a, b, c, d, wd_t, MD5C1c, MD5S10); - MD5_STEP (MD5_Go, d, a, b, c, w2_t, MD5C1d, MD5S11); - MD5_STEP (MD5_Go, c, d, a, b, w7_t, MD5C1e, MD5S12); - MD5_STEP (MD5_Go, b, c, d, a, wc_t, MD5C1f, MD5S13); - - MD5_STEP (MD5_H , a, b, c, d, w5_t, MD5C20, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w8_t, MD5C21, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wb_t, MD5C22, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, we_t, MD5C23, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w1_t, MD5C24, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w4_t, MD5C25, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w7_t, MD5C26, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, wa_t, MD5C27, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, wd_t, MD5C28, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, w0_t, MD5C29, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, w3_t, MD5C2a, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w6_t, MD5C2b, MD5S23); - MD5_STEP (MD5_H , a, b, c, d, w9_t, MD5C2c, MD5S20); - MD5_STEP (MD5_H , d, a, b, c, wc_t, MD5C2d, MD5S21); - MD5_STEP (MD5_H , c, d, a, b, wf_t, MD5C2e, MD5S22); - MD5_STEP (MD5_H , b, c, d, a, w2_t, MD5C2f, MD5S23); - - MD5_STEP (MD5_I , a, b, c, d, w0_t, MD5C30, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w7_t, MD5C31, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, we_t, MD5C32, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w5_t, MD5C33, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, wc_t, MD5C34, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, w3_t, MD5C35, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, wa_t, MD5C36, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w1_t, MD5C37, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w8_t, MD5C38, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wf_t, MD5C39, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w6_t, MD5C3a, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, wd_t, MD5C3b, MD5S33); - MD5_STEP (MD5_I , a, b, c, d, w4_t, MD5C3c, MD5S30); - MD5_STEP (MD5_I , d, a, b, c, wb_t, MD5C3d, MD5S31); - MD5_STEP (MD5_I , c, d, a, b, w2_t, MD5C3e, MD5S32); - MD5_STEP (MD5_I , b, c, d, a, w9_t, MD5C3f, MD5S33); - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; -} - -void memcat16 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat16_x80 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[4]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0x80; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0x80, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - block0[3] = tmp3; - block1[0] = tmp4; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - block1[0] = tmp3; - block1[1] = tmp4; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - block1[1] = tmp3; - block1[2] = tmp4; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - block1[2] = tmp3; - block1[3] = tmp4; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - block1[3] = tmp3; - block2[0] = tmp4; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - block2[0] = tmp3; - block2[1] = tmp4; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - block2[1] = tmp3; - block2[2] = tmp4; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - block2[2] = tmp3; - block2[3] = tmp4; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - block2[3] = tmp3; - block3[0] = tmp4; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - block3[0] = tmp3; - block3[1] = tmp4; - break; - } -} - -void memcat8 (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append[2]) -{ - u32 tmp0; - u32 tmp1; - u32 tmp2; - - #if defined IS_AMD || defined IS_GENERIC - - const int offset_minus_4 = 4 - (block_len & 3); - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign ( 0, append[1], offset_minus_4); - - const u32 mod = block_len & 3; - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = 0; - } - - #endif - - #ifdef IS_NV - - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], 0, selector); - - #endif - - const u32 div = block_len / 4; - - switch (div) - { - case 0: block0[0] |= tmp0; - block0[1] = tmp1; - block0[2] = tmp2; - break; - case 1: block0[1] |= tmp0; - block0[2] = tmp1; - block0[3] = tmp2; - break; - case 2: block0[2] |= tmp0; - block0[3] = tmp1; - block1[0] = tmp2; - break; - case 3: block0[3] |= tmp0; - block1[0] = tmp1; - block1[1] = tmp2; - break; - case 4: block1[0] |= tmp0; - block1[1] = tmp1; - block1[2] = tmp2; - break; - case 5: block1[1] |= tmp0; - block1[2] = tmp1; - block1[3] = tmp2; - break; - case 6: block1[2] |= tmp0; - block1[3] = tmp1; - block2[0] = tmp2; - break; - case 7: block1[3] |= tmp0; - block2[0] = tmp1; - block2[1] = tmp2; - break; - case 8: block2[0] |= tmp0; - block2[1] = tmp1; - block2[2] = tmp2; - break; - case 9: block2[1] |= tmp0; - block2[2] = tmp1; - block2[3] = tmp2; - break; - case 10: block2[2] |= tmp0; - block2[3] = tmp1; - block3[0] = tmp2; - break; - case 11: block2[3] |= tmp0; - block3[0] = tmp1; - block3[1] = tmp2; - break; - } -} - -void append_1st (u32 block0[4], u32 block1[4], u32 block2[4], u32 block3[4], const u32 block_len, const u32 append) -{ - switch (block_len) - { - case 0: - block0[0] = append; - break; - - case 1: - block0[0] = block0[0] | append << 8; - break; - - case 2: - block0[0] = block0[0] | append << 16; - break; - - case 3: - block0[0] = block0[0] | append << 24; - break; - - case 4: - block0[1] = append; - break; - - case 5: - block0[1] = block0[1] | append << 8; - break; - - case 6: - block0[1] = block0[1] | append << 16; - break; - - case 7: - block0[1] = block0[1] | append << 24; - break; - - case 8: - block0[2] = append; - break; - - case 9: - block0[2] = block0[2] | append << 8; - break; - - case 10: - block0[2] = block0[2] | append << 16; - break; - - case 11: - block0[2] = block0[2] | append << 24; - break; - - case 12: - block0[3] = append; - break; - - case 13: - block0[3] = block0[3] | append << 8; - break; - - case 14: - block0[3] = block0[3] | append << 16; - break; - - case 15: - block0[3] = block0[3] | append << 24; - break; - - case 16: - block1[0] = append; - break; - - case 17: - block1[0] = block1[0] | append << 8; - break; - - case 18: - block1[0] = block1[0] | append << 16; - break; - - case 19: - block1[0] = block1[0] | append << 24; - break; - - case 20: - block1[1] = append; - break; - - case 21: - block1[1] = block1[1] | append << 8; - break; - - case 22: - block1[1] = block1[1] | append << 16; - break; - - case 23: - block1[1] = block1[1] | append << 24; - break; - - case 24: - block1[2] = append; - break; - - case 25: - block1[2] = block1[2] | append << 8; - break; - - case 26: - block1[2] = block1[2] | append << 16; - break; - - case 27: - block1[2] = block1[2] | append << 24; - break; - - case 28: - block1[3] = append; - break; - - case 29: - block1[3] = block1[3] | append << 8; - break; - - case 30: - block1[3] = block1[3] | append << 16; - break; - - case 31: - block1[3] = block1[3] | append << 24; - break; - - case 32: - block2[0] = append; - break; - - case 33: - block2[0] = block2[0] | append << 8; - break; - - case 34: - block2[0] = block2[0] | append << 16; - break; - - case 35: - block2[0] = block2[0] | append << 24; - break; - - case 36: - block2[1] = append; - break; - - case 37: - block2[1] = block2[1] | append << 8; - break; - - case 38: - block2[1] = block2[1] | append << 16; - break; - - case 39: - block2[1] = block2[1] | append << 24; - break; - - case 40: - block2[2] = append; - break; - - case 41: - block2[2] = block2[2] | append << 8; - break; - - case 42: - block2[2] = block2[2] | append << 16; - break; - - case 43: - block2[2] = block2[2] | append << 24; - break; - - case 44: - block2[3] = append; - break; - - case 45: - block2[3] = block2[3] | append << 8; - break; - - case 46: - block2[3] = block2[3] | append << 16; - break; - - case 47: - block2[3] = block2[3] | append << 24; - break; - - case 48: - block3[0] = append; - break; - - case 49: - block3[0] = block3[0] | append << 8; - break; - - case 50: - block3[0] = block3[0] | append << 16; - break; - - case 51: - block3[0] = block3[0] | append << 24; - break; - - case 52: - block3[1] = append; - break; - - case 53: - block3[1] = block3[1] | append << 8; - break; - - case 54: - block3[1] = block3[1] | append << 16; - break; - - case 55: - block3[1] = block3[1] | append << 24; - break; - - case 56: - block3[2] = append; - break; - } -} - __kernel void m06300_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -670,152 +25,102 @@ __kernel void m06300_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - /** * init */ - //memcat16 (block0, block1, block2, block3, block_len, w0); - //block_len += pw_len; + const u32 pw_len = pws[gid].pw_len; - u32 block_len = pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 block0[4]; + u32 w[64] = { 0 }; - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - u32 block1[4]; + barrier (CLK_GLOBAL_MEM_FENCE); + } - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 block2[4]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + u32 s[64] = { 0 }; - u32 block3[4]; + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + barrier (CLK_GLOBAL_MEM_FENCE); + } - memcat8 (block0, block1, block2, block3, block_len, salt_buf); + /** + * prepare + */ - block_len += salt_len; + md5_ctx_t md5_ctx1; - memcat16 (block0, block1, block2, block3, block_len, w0); + md5_init (&md5_ctx1); - block_len += pw_len; + md5_update (&md5_ctx1, w, pw_len); - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_update (&md5_ctx1, s, salt_len); - block3[2] = block_len * 8; + md5_update (&md5_ctx1, w, pw_len); - u32 digest[4]; + md5_final (&md5_ctx1); - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + u32 final[16] = { 0 }; - md5_transform (block0, block1, block2, block3, digest); + final[0] = md5_ctx1.h[0]; + final[1] = md5_ctx1.h[1]; + final[2] = md5_ctx1.h[2]; + final[3] = md5_ctx1.h[3]; - /* The password first, since that is what is most unknown */ - /* Then the raw salt */ - /* Then just as many characters of the MD5(pw,salt,pw) */ + md5_ctx_t md5_ctx; - //memcat16 (block0, block1, block2, block3, block_len, w); - //block_len += pw_len; + md5_init (&md5_ctx); - block_len = pw_len; + md5_update (&md5_ctx, w, pw_len); - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; + md5_update (&md5_ctx, s, salt_len); - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; + int pl; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; + for (pl = pw_len; pl > 16; pl -= 16) + { + md5_update (&md5_ctx, final, 16); + } - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; + truncate_block_4x4_le (final, pl); - memcat8 (block0, block1, block2, block3, block_len, salt_buf); - - block_len += salt_len; - - truncate_block_4x4_le (digest, pw_len); - - memcat16 (block0, block1, block2, block3, block_len, digest); - - block_len += pw_len; + md5_update (&md5_ctx, final, pl); /* Then something really weird... */ - u32 append = block0[0] & 0xFF; - - for (u32 j = pw_len; j; j >>= 1) + for (int i = pw_len; i != 0; i >>= 1) { - if ((j & 1) == 0) + u32 t[16] = { 0 }; + + if (i & 1) { - append_1st (block0, block1, block2, block3, block_len, append); + t[0] = 0; + } + else + { + t[0] = w[0] & 0xff; } - block_len++; + md5_update (&md5_ctx, t, 1); } - append_0x80_4x4 (block0, block1, block2, block3, block_len); + md5_final (&md5_ctx); - block3[2] = block_len * 8; - - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; - - md5_transform (block0, block1, block2, block3, digest); - - tmps[gid].digest_buf[0] = digest[0]; - tmps[gid].digest_buf[1] = digest[1]; - tmps[gid].digest_buf[2] = digest[2]; - tmps[gid].digest_buf[3] = digest[3]; + tmps[gid].digest_buf[0] = md5_ctx.h[0]; + tmps[gid].digest_buf[1] = md5_ctx.h[1]; + tmps[gid].digest_buf[2] = md5_ctx.h[2]; + tmps[gid].digest_buf[3] = md5_ctx.h[3]; } __kernel void m06300_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global md5crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -828,40 +133,41 @@ __kernel void m06300_loop (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; + /** + * init + */ const u32 pw_len = pws[gid].pw_len; - u32 w0_x80[4]; + const u32 pw_lenv = ceil ((float) pw_len / 4); - w0_x80[0] = w0[0]; - w0_x80[1] = w0[1]; - w0_x80[2] = w0[2]; - w0_x80[3] = w0[3]; + u32 w[64] = { 0 }; - append_0x80_1x4 (w0_x80, pw_len); + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; - /** - * salt - */ - - u32 salt_buf[2]; - - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + barrier (CLK_GLOBAL_MEM_FENCE); + } const u32 salt_len = salt_bufs[salt_pos].salt_len; + const u32 salt_lenv = ceil ((float) salt_len / 4); + + u32 s[64] = { 0 }; + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * digest */ - u32 digest[4]; + u32 digest[16] = { 0 }; digest[0] = tmps[gid].digest_buf[0]; digest[1] = tmps[gid].digest_buf[1]; @@ -872,132 +178,46 @@ __kernel void m06300_loop (__global pw_t *pws, __global const kernel_rule_t *rul * loop */ - /* and now, just to make sure things don't run too fast */ - - u32 block_len; - - u32 block0[4]; - - block0[0] = 0; - block0[1] = 0; - block0[2] = 0; - block0[3] = 0; - - u32 block1[4]; - - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - - u32 block2[4]; - - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - - u32 block3[4]; - - block3[0] = 0; - block3[1] = 0; - block3[2] = 0; - block3[3] = 0; - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - block1[0] = 0; - block1[1] = 0; - block1[2] = 0; - block1[3] = 0; - block2[0] = 0; - block2[1] = 0; - block2[2] = 0; - block2[3] = 0; - block3[0] = 0; - block3[1] = 0; + md5_ctx_t md5_ctx; - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 1 : 0; - const u32 j7 = (j % 7) ? 1 : 0; + md5_init (&md5_ctx); - if (j1) + if (j & 1) { - block0[0] = w0[0]; - block0[1] = w0[1]; - block0[2] = w0[2]; - block0[3] = w0[3]; - - block_len = pw_len; - - if (j3) - { - memcat8 (block0, block1, block2, block3, block_len, salt_buf); - - block_len += salt_len; - } - - if (j7) - { - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - - memcat16_x80 (block0, block1, block2, block3, block_len, digest); - - block_len += 16; + md5_update (&md5_ctx, w, pw_len); } - else + else { - block0[0] = digest[0]; - block0[1] = digest[1]; - block0[2] = digest[2]; - block0[3] = digest[3]; - - block_len = 16; - - if (j3 && j7) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - - memcat16 (block0, block1, block2, block3, block_len, w0); - - block_len += pw_len; - } - else if (j3) - { - block1[0] = salt_buf[0]; - block1[1] = salt_buf[1]; - - block_len += salt_len; - } - else if (j7) - { - block1[0] = w0[0]; - block1[1] = w0[1]; - block1[2] = w0[2]; - block1[3] = w0[3]; - - block_len += pw_len; - } - - memcat16 (block0, block1, block2, block3, block_len, w0_x80); - - block_len += pw_len; + md5_update (&md5_ctx, digest, 16); } - block3[2] = block_len * 8; + if (j % 3) + { + md5_update (&md5_ctx, s, salt_len); + } - digest[0] = MD5M_A; - digest[1] = MD5M_B; - digest[2] = MD5M_C; - digest[3] = MD5M_D; + if (j % 7) + { + md5_update (&md5_ctx, w, pw_len); + } - md5_transform (block0, block1, block2, block3, digest); + if (j & 1) + { + md5_update (&md5_ctx, digest, 16); + } + else + { + md5_update (&md5_ctx, w, pw_len); + } + + md5_final (&md5_ctx); + + digest[0] = md5_ctx.h[0]; + digest[1] = md5_ctx.h[1]; + digest[2] = md5_ctx.h[2]; + digest[3] = md5_ctx.h[3]; } tmps[gid].digest_buf[0] = digest[0]; diff --git a/OpenCL/m07400-optimized.cl b/OpenCL/m07400-optimized.cl new file mode 100644 index 000000000..e3a9fbda5 --- /dev/null +++ b/OpenCL/m07400-optimized.cl @@ -0,0 +1,1755 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +__constant u32a k_sha256[64] = +{ + SHA256C00, SHA256C01, SHA256C02, SHA256C03, + SHA256C04, SHA256C05, SHA256C06, SHA256C07, + SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, + SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, + SHA256C10, SHA256C11, SHA256C12, SHA256C13, + SHA256C14, SHA256C15, SHA256C16, SHA256C17, + SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, + SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, + SHA256C20, SHA256C21, SHA256C22, SHA256C23, + SHA256C24, SHA256C25, SHA256C26, SHA256C27, + SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, + SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, + SHA256C30, SHA256C31, SHA256C32, SHA256C33, + SHA256C34, SHA256C35, SHA256C36, SHA256C37, + SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, + SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, +}; + +#if 1 + +void sha256_transform (const u32 w[16], u32 digest[8]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + u32 e = digest[4]; + u32 f = digest[5]; + u32 g = digest[6]; + u32 h = digest[7]; + + u32 w0_t = swap32 (w[ 0]); + u32 w1_t = swap32 (w[ 1]); + u32 w2_t = swap32 (w[ 2]); + u32 w3_t = swap32 (w[ 3]); + u32 w4_t = swap32 (w[ 4]); + u32 w5_t = swap32 (w[ 5]); + u32 w6_t = swap32 (w[ 6]); + u32 w7_t = swap32 (w[ 7]); + u32 w8_t = swap32 (w[ 8]); + u32 w9_t = swap32 (w[ 9]); + u32 wa_t = swap32 (w[10]); + u32 wb_t = swap32 (w[11]); + u32 wc_t = swap32 (w[12]); + u32 wd_t = swap32 (w[13]); + u32 we_t = swap32 (w[14]); + u32 wf_t = swap32 (w[15]); + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void sha256_transform_no14 (const u32 w[16], u32 digest[8]) +{ + u32 w_t[16]; + + w_t[ 0] = w[ 0]; + w_t[ 1] = w[ 1]; + w_t[ 2] = w[ 2]; + w_t[ 3] = w[ 3]; + w_t[ 4] = w[ 4]; + w_t[ 5] = w[ 5]; + w_t[ 6] = w[ 6]; + w_t[ 7] = w[ 7]; + w_t[ 8] = w[ 8]; + w_t[ 9] = w[ 9]; + w_t[10] = w[10]; + w_t[11] = w[11]; + w_t[12] = w[12]; + w_t[13] = w[13]; + w_t[14] = 0; + w_t[15] = w[15]; + + sha256_transform (w_t, digest); +} + +void init_ctx (u32 digest[8]) +{ + digest[0] = SHA256M_A; + digest[1] = SHA256M_B; + digest[2] = SHA256M_C; + digest[3] = SHA256M_D; + digest[4] = SHA256M_E; + digest[5] = SHA256M_F; + digest[6] = SHA256M_G; + digest[7] = SHA256M_H; +} + +void bzero16 (u32 block[16]) +{ + block[ 0] = 0; + block[ 1] = 0; + block[ 2] = 0; + block[ 3] = 0; + block[ 4] = 0; + block[ 5] = 0; + block[ 6] = 0; + block[ 7] = 0; + block[ 8] = 0; + block[ 9] = 0; + block[10] = 0; + block[11] = 0; + block[12] = 0; + block[13] = 0; + block[14] = 0; + block[15] = 0; +} + +void bswap8 (u32 block[16]) +{ + block[ 0] = swap32 (block[ 0]); + block[ 1] = swap32 (block[ 1]); + block[ 2] = swap32 (block[ 2]); + block[ 3] = swap32 (block[ 3]); + block[ 4] = swap32 (block[ 4]); + block[ 5] = swap32 (block[ 5]); + block[ 6] = swap32 (block[ 6]); + block[ 7] = swap32 (block[ 7]); +} + +u32 memcat16 (u32 block[16], const u32 block_len, const u32 append[4], const u32 append_len) +{ + const u32 mod = block_len & 3; + const u32 div = block_len / 4; + + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + const int offset_minus_4 = 4 - block_len; + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + #endif + + switch (div) + { + case 0: block[ 0] |= tmp0; + block[ 1] = tmp1; + block[ 2] = tmp2; + block[ 3] = tmp3; + block[ 4] = tmp4; + break; + case 1: block[ 1] |= tmp0; + block[ 2] = tmp1; + block[ 3] = tmp2; + block[ 4] = tmp3; + block[ 5] = tmp4; + break; + case 2: block[ 2] |= tmp0; + block[ 3] = tmp1; + block[ 4] = tmp2; + block[ 5] = tmp3; + block[ 6] = tmp4; + break; + case 3: block[ 3] |= tmp0; + block[ 4] = tmp1; + block[ 5] = tmp2; + block[ 6] = tmp3; + block[ 7] = tmp4; + break; + case 4: block[ 4] |= tmp0; + block[ 5] = tmp1; + block[ 6] = tmp2; + block[ 7] = tmp3; + block[ 8] = tmp4; + break; + case 5: block[ 5] |= tmp0; + block[ 6] = tmp1; + block[ 7] = tmp2; + block[ 8] = tmp3; + block[ 9] = tmp4; + break; + case 6: block[ 6] |= tmp0; + block[ 7] = tmp1; + block[ 8] = tmp2; + block[ 9] = tmp3; + block[10] = tmp4; + break; + case 7: block[ 7] |= tmp0; + block[ 8] = tmp1; + block[ 9] = tmp2; + block[10] = tmp3; + block[11] = tmp4; + break; + case 8: block[ 8] |= tmp0; + block[ 9] = tmp1; + block[10] = tmp2; + block[11] = tmp3; + block[12] = tmp4; + break; + case 9: block[ 9] |= tmp0; + block[10] = tmp1; + block[11] = tmp2; + block[12] = tmp3; + block[13] = tmp4; + break; + case 10: block[10] |= tmp0; + block[11] = tmp1; + block[12] = tmp2; + block[13] = tmp3; + block[14] = tmp4; + break; + case 11: block[11] |= tmp0; + block[12] = tmp1; + block[13] = tmp2; + block[14] = tmp3; + block[15] = tmp4; + break; + case 12: block[12] |= tmp0; + block[13] = tmp1; + block[14] = tmp2; + block[15] = tmp3; + break; + case 13: block[13] |= tmp0; + block[14] = tmp1; + block[15] = tmp2; + break; + case 14: block[14] |= tmp0; + block[15] = tmp1; + break; + case 15: block[15] |= tmp0; + break; + } + + u32 new_len = block_len + append_len; + + return new_len; +} + +u32 memcat16c (u32 block[16], const u32 block_len, const u32 append[4], const u32 append_len, u32 digest[8]) +{ + const u32 mod = block_len & 3; + const u32 div = block_len / 4; + + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + const int offset_minus_4 = 4 - block_len; + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + #endif + + u32 carry[4] = { 0, 0, 0, 0 }; + + switch (div) + { + case 0: block[ 0] |= tmp0; + block[ 1] = tmp1; + block[ 2] = tmp2; + block[ 3] = tmp3; + block[ 4] = tmp4; + break; + case 1: block[ 1] |= tmp0; + block[ 2] = tmp1; + block[ 3] = tmp2; + block[ 4] = tmp3; + block[ 5] = tmp4; + break; + case 2: block[ 2] |= tmp0; + block[ 3] = tmp1; + block[ 4] = tmp2; + block[ 5] = tmp3; + block[ 6] = tmp4; + break; + case 3: block[ 3] |= tmp0; + block[ 4] = tmp1; + block[ 5] = tmp2; + block[ 6] = tmp3; + block[ 7] = tmp4; + break; + case 4: block[ 4] |= tmp0; + block[ 5] = tmp1; + block[ 6] = tmp2; + block[ 7] = tmp3; + block[ 8] = tmp4; + break; + case 5: block[ 5] |= tmp0; + block[ 6] = tmp1; + block[ 7] = tmp2; + block[ 8] = tmp3; + block[ 9] = tmp4; + break; + case 6: block[ 6] |= tmp0; + block[ 7] = tmp1; + block[ 8] = tmp2; + block[ 9] = tmp3; + block[10] = tmp4; + break; + case 7: block[ 7] |= tmp0; + block[ 8] = tmp1; + block[ 9] = tmp2; + block[10] = tmp3; + block[11] = tmp4; + break; + case 8: block[ 8] |= tmp0; + block[ 9] = tmp1; + block[10] = tmp2; + block[11] = tmp3; + block[12] = tmp4; + break; + case 9: block[ 9] |= tmp0; + block[10] = tmp1; + block[11] = tmp2; + block[12] = tmp3; + block[13] = tmp4; + break; + case 10: block[10] |= tmp0; + block[11] = tmp1; + block[12] = tmp2; + block[13] = tmp3; + block[14] = tmp4; + break; + case 11: block[11] |= tmp0; + block[12] = tmp1; + block[13] = tmp2; + block[14] = tmp3; + block[15] = tmp4; + break; + case 12: block[12] |= tmp0; + block[13] = tmp1; + block[14] = tmp2; + block[15] = tmp3; + carry[ 0] = tmp4; + break; + case 13: block[13] |= tmp0; + block[14] = tmp1; + block[15] = tmp2; + carry[ 0] = tmp3; + carry[ 1] = tmp4; + break; + case 14: block[14] |= tmp0; + block[15] = tmp1; + carry[ 0] = tmp2; + carry[ 1] = tmp3; + carry[ 2] = tmp4; + break; + case 15: block[15] |= tmp0; + carry[ 0] = tmp1; + carry[ 1] = tmp2; + carry[ 2] = tmp3; + carry[ 3] = tmp4; + break; + } + + u32 new_len = block_len + append_len; + + if (new_len >= 64) + { + new_len -= 64; + + sha256_transform (block, digest); + + bzero16 (block); + + block[0] = carry[0]; + block[1] = carry[1]; + block[2] = carry[2]; + block[3] = carry[3]; + } + + return new_len; +} + +u32 memcat20 (u32 block[20], const u32 block_len, const u32 append[4], const u32 append_len) +{ + const u32 mod = block_len & 3; + const u32 div = block_len / 4; + + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + const int offset_minus_4 = 4 - block_len; + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0, selector); + #endif + + switch (div) + { + case 0: block[ 0] |= tmp0; + block[ 1] = tmp1; + block[ 2] = tmp2; + block[ 3] = tmp3; + block[ 4] = tmp4; + break; + case 1: block[ 1] |= tmp0; + block[ 2] = tmp1; + block[ 3] = tmp2; + block[ 4] = tmp3; + block[ 5] = tmp4; + break; + case 2: block[ 2] |= tmp0; + block[ 3] = tmp1; + block[ 4] = tmp2; + block[ 5] = tmp3; + block[ 6] = tmp4; + break; + case 3: block[ 3] |= tmp0; + block[ 4] = tmp1; + block[ 5] = tmp2; + block[ 6] = tmp3; + block[ 7] = tmp4; + break; + case 4: block[ 4] |= tmp0; + block[ 5] = tmp1; + block[ 6] = tmp2; + block[ 7] = tmp3; + block[ 8] = tmp4; + break; + case 5: block[ 5] |= tmp0; + block[ 6] = tmp1; + block[ 7] = tmp2; + block[ 8] = tmp3; + block[ 9] = tmp4; + break; + case 6: block[ 6] |= tmp0; + block[ 7] = tmp1; + block[ 8] = tmp2; + block[ 9] = tmp3; + block[10] = tmp4; + break; + case 7: block[ 7] |= tmp0; + block[ 8] = tmp1; + block[ 9] = tmp2; + block[10] = tmp3; + block[11] = tmp4; + break; + case 8: block[ 8] |= tmp0; + block[ 9] = tmp1; + block[10] = tmp2; + block[11] = tmp3; + block[12] = tmp4; + break; + case 9: block[ 9] |= tmp0; + block[10] = tmp1; + block[11] = tmp2; + block[12] = tmp3; + block[13] = tmp4; + break; + case 10: block[10] |= tmp0; + block[11] = tmp1; + block[12] = tmp2; + block[13] = tmp3; + block[14] = tmp4; + break; + case 11: block[11] |= tmp0; + block[12] = tmp1; + block[13] = tmp2; + block[14] = tmp3; + block[15] = tmp4; + break; + case 12: block[12] |= tmp0; + block[13] = tmp1; + block[14] = tmp2; + block[15] = tmp3; + block[16] = tmp4; + break; + case 13: block[13] |= tmp0; + block[14] = tmp1; + block[15] = tmp2; + block[16] = tmp3; + block[17] = tmp4; + break; + case 14: block[14] |= tmp0; + block[15] = tmp1; + block[16] = tmp2; + block[17] = tmp3; + block[18] = tmp4; + break; + case 15: block[15] |= tmp0; + block[16] = tmp1; + block[17] = tmp2; + block[18] = tmp3; + block[19] = tmp4; + break; + } + + return block_len + append_len; +} + +u32 memcat20_x80 (u32 block[20], const u32 block_len, const u32 append[4], const u32 append_len) +{ + const u32 mod = block_len & 3; + const u32 div = block_len / 4; + + u32 tmp0; + u32 tmp1; + u32 tmp2; + u32 tmp3; + u32 tmp4; + + #if defined IS_AMD || defined IS_GENERIC + const int offset_minus_4 = 4 - block_len; + + tmp0 = amd_bytealign (append[0], 0, offset_minus_4); + tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); + tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); + tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); + tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); + + if (mod == 0) + { + tmp0 = tmp1; + tmp1 = tmp2; + tmp2 = tmp3; + tmp3 = tmp4; + tmp4 = 0x80; + } + #endif + + #ifdef IS_NV + const int offset_minus_4 = 4 - (block_len & 3); + + const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; + + tmp0 = __byte_perm ( 0, append[0], selector); + tmp1 = __byte_perm (append[0], append[1], selector); + tmp2 = __byte_perm (append[1], append[2], selector); + tmp3 = __byte_perm (append[2], append[3], selector); + tmp4 = __byte_perm (append[3], 0x80, selector); + #endif + + switch (div) + { + case 0: block[ 0] |= tmp0; + block[ 1] = tmp1; + block[ 2] = tmp2; + block[ 3] = tmp3; + block[ 4] = tmp4; + break; + case 1: block[ 1] |= tmp0; + block[ 2] = tmp1; + block[ 3] = tmp2; + block[ 4] = tmp3; + block[ 5] = tmp4; + break; + case 2: block[ 2] |= tmp0; + block[ 3] = tmp1; + block[ 4] = tmp2; + block[ 5] = tmp3; + block[ 6] = tmp4; + break; + case 3: block[ 3] |= tmp0; + block[ 4] = tmp1; + block[ 5] = tmp2; + block[ 6] = tmp3; + block[ 7] = tmp4; + break; + case 4: block[ 4] |= tmp0; + block[ 5] = tmp1; + block[ 6] = tmp2; + block[ 7] = tmp3; + block[ 8] = tmp4; + break; + case 5: block[ 5] |= tmp0; + block[ 6] = tmp1; + block[ 7] = tmp2; + block[ 8] = tmp3; + block[ 9] = tmp4; + break; + case 6: block[ 6] |= tmp0; + block[ 7] = tmp1; + block[ 8] = tmp2; + block[ 9] = tmp3; + block[10] = tmp4; + break; + case 7: block[ 7] |= tmp0; + block[ 8] = tmp1; + block[ 9] = tmp2; + block[10] = tmp3; + block[11] = tmp4; + break; + case 8: block[ 8] |= tmp0; + block[ 9] = tmp1; + block[10] = tmp2; + block[11] = tmp3; + block[12] = tmp4; + break; + case 9: block[ 9] |= tmp0; + block[10] = tmp1; + block[11] = tmp2; + block[12] = tmp3; + block[13] = tmp4; + break; + case 10: block[10] |= tmp0; + block[11] = tmp1; + block[12] = tmp2; + block[13] = tmp3; + block[14] = tmp4; + break; + case 11: block[11] |= tmp0; + block[12] = tmp1; + block[13] = tmp2; + block[14] = tmp3; + block[15] = tmp4; + break; + case 12: block[12] |= tmp0; + block[13] = tmp1; + block[14] = tmp2; + block[15] = tmp3; + block[16] = tmp4; + break; + case 13: block[13] |= tmp0; + block[14] = tmp1; + block[15] = tmp2; + block[16] = tmp3; + block[17] = tmp4; + break; + case 14: block[14] |= tmp0; + block[15] = tmp1; + block[16] = tmp2; + block[17] = tmp3; + block[18] = tmp4; + break; + case 15: block[15] |= tmp0; + block[16] = tmp1; + block[17] = tmp2; + block[18] = tmp3; + block[19] = tmp4; + break; + } + + return block_len + append_len; +} + +__kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt_buf[4]; + + salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; + salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; + salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; + salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; + + u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * buffers + */ + + u32 block_len; // never reaches > 64 + u32 transform_len; // required for w[15] = len * 8 + + u32 block[16]; + + u32 alt_result[8]; + u32 p_bytes[8]; + u32 s_bytes[8]; + + /* Prepare for the real work. */ + + block_len = 0; + + bzero16 (block); + + /* Add key. */ + + block_len = memcat16 (block, block_len, w0, pw_len); + + /* Add salt. */ + + block_len = memcat16 (block, block_len, salt_buf, salt_len); + + /* Add key again. */ + + block_len = memcat16 (block, block_len, w0, pw_len); + + append_0x80_1x16 (block, block_len); + + block[15] = swap32 (block_len * 8); + + init_ctx (alt_result); + + sha256_transform (block, alt_result); + + bswap8 (alt_result); + + block_len = 0; + + bzero16 (block); + + u32 alt_result_tmp[8]; + + alt_result_tmp[0] = alt_result[0]; + alt_result_tmp[1] = alt_result[1]; + alt_result_tmp[2] = alt_result[2]; + alt_result_tmp[3] = alt_result[3]; + alt_result_tmp[4] = 0; + alt_result_tmp[5] = 0; + alt_result_tmp[6] = 0; + alt_result_tmp[7] = 0; + + truncate_block_4x4_le (alt_result_tmp, pw_len); + + /* Add the key string. */ + + block_len = memcat16 (block, block_len, w0, pw_len); + + /* The last part is the salt string. This must be at most 8 + characters and it ends at the first `$' character (for + compatibility with existing implementations). */ + + block_len = memcat16 (block, block_len, salt_buf, salt_len); + + /* Now get result of this (32 bytes) and add it to the other + context. */ + + block_len = memcat16 (block, block_len, alt_result_tmp, pw_len); + + transform_len = block_len; + + /* Take the binary representation of the length of the key and for every + 1 add the alternate sum, for every 0 the key. */ + + alt_result_tmp[0] = alt_result[0]; + alt_result_tmp[1] = alt_result[1]; + alt_result_tmp[2] = alt_result[2]; + alt_result_tmp[3] = alt_result[3]; + alt_result_tmp[4] = alt_result[4]; + alt_result_tmp[5] = alt_result[5]; + alt_result_tmp[6] = alt_result[6]; + alt_result_tmp[7] = alt_result[7]; + + init_ctx (alt_result); + + for (u32 j = pw_len; j; j >>= 1) + { + if (j & 1) + { + block_len = memcat16c (block, block_len, &alt_result_tmp[0], 16, alt_result); + block_len = memcat16c (block, block_len, &alt_result_tmp[4], 16, alt_result); + + transform_len += 32; + } + else + { + block_len = memcat16c (block, block_len, w0, pw_len, alt_result); + + transform_len += pw_len; + } + } + + append_0x80_1x16 (block, block_len); + + if (block_len >= 56) + { + sha256_transform (block, alt_result); + + bzero16 (block); + } + + block[15] = swap32 (transform_len * 8); + + sha256_transform (block, alt_result); + + bswap8 (alt_result); + + tmps[gid].alt_result[0] = alt_result[0]; + tmps[gid].alt_result[1] = alt_result[1]; + tmps[gid].alt_result[2] = alt_result[2]; + tmps[gid].alt_result[3] = alt_result[3]; + tmps[gid].alt_result[4] = alt_result[4]; + tmps[gid].alt_result[5] = alt_result[5]; + tmps[gid].alt_result[6] = alt_result[6]; + tmps[gid].alt_result[7] = alt_result[7]; + + /* Start computation of P byte sequence. */ + + block_len = 0; + + transform_len = 0; + + bzero16 (block); + + /* For every character in the password add the entire password. */ + + init_ctx (p_bytes); + + for (u32 j = 0; j < pw_len; j++) + { + block_len = memcat16c (block, block_len, w0, pw_len, p_bytes); + + transform_len += pw_len; + } + + /* Finish the digest. */ + + append_0x80_1x16 (block, block_len); + + if (block_len >= 56) + { + sha256_transform (block, p_bytes); + + bzero16 (block); + } + + block[15] = swap32 (transform_len * 8); + + sha256_transform (block, p_bytes); + + bswap8 (p_bytes); + + truncate_block_4x4_le (p_bytes, pw_len); + + tmps[gid].p_bytes[0] = p_bytes[0]; + tmps[gid].p_bytes[1] = p_bytes[1]; + tmps[gid].p_bytes[2] = p_bytes[2]; + tmps[gid].p_bytes[3] = p_bytes[3]; + + /* Start computation of S byte sequence. */ + + block_len = 0; + + transform_len = 0; + + bzero16 (block); + + /* For every character in the password add the entire password. */ + + init_ctx (s_bytes); + + for (u32 j = 0; j < 16 + (alt_result[0] & 0xff); j++) + { + block_len = memcat16c (block, block_len, salt_buf, salt_len, s_bytes); + + transform_len += salt_len; + } + + /* Finish the digest. */ + + append_0x80_1x16 (block, block_len); + + if (block_len >= 56) + { + sha256_transform (block, s_bytes); + + bzero16 (block); + } + + block[15] = swap32 (transform_len * 8); + + sha256_transform (block, s_bytes); + + bswap8 (s_bytes); + + truncate_block_4x4_le (s_bytes, salt_len); + + tmps[gid].s_bytes[0] = s_bytes[0]; + tmps[gid].s_bytes[1] = s_bytes[1]; + tmps[gid].s_bytes[2] = s_bytes[2]; + tmps[gid].s_bytes[3] = s_bytes[3]; +} + +__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 pw_len = pws[gid].pw_len; + + /** + * base + */ + + u32 p_bytes[4]; + + p_bytes[0] = tmps[gid].p_bytes[0]; + p_bytes[1] = tmps[gid].p_bytes[1]; + p_bytes[2] = tmps[gid].p_bytes[2]; + p_bytes[3] = tmps[gid].p_bytes[3]; + + u32 p_bytes_x80[4]; + + p_bytes_x80[0] = tmps[gid].p_bytes[0]; + p_bytes_x80[1] = tmps[gid].p_bytes[1]; + p_bytes_x80[2] = tmps[gid].p_bytes[2]; + p_bytes_x80[3] = tmps[gid].p_bytes[3]; + + append_0x80_1x4 (p_bytes_x80, pw_len); + + u32 s_bytes[4]; + + s_bytes[0] = tmps[gid].s_bytes[0]; + s_bytes[1] = tmps[gid].s_bytes[1]; + s_bytes[2] = tmps[gid].s_bytes[2]; + s_bytes[3] = tmps[gid].s_bytes[3]; + + u32 alt_result[8]; + + alt_result[0] = tmps[gid].alt_result[0]; + alt_result[1] = tmps[gid].alt_result[1]; + alt_result[2] = tmps[gid].alt_result[2]; + alt_result[3] = tmps[gid].alt_result[3]; + alt_result[4] = tmps[gid].alt_result[4]; + alt_result[5] = tmps[gid].alt_result[5]; + alt_result[6] = tmps[gid].alt_result[6]; + alt_result[7] = tmps[gid].alt_result[7]; + + u32 salt_len = salt_bufs[salt_pos].salt_len; + + /* Repeatedly run the collected hash value through SHA256 to burn + CPU cycles. */ + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + u32 tmp[8]; + + init_ctx (tmp); + + u32 block[32]; + + bzero16 (&block[ 0]); + bzero16 (&block[16]); + + u32 block_len = 0; + + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 1 : 0; + const u32 j7 = (j % 7) ? 1 : 0; + + if (j1) + { + block[0] = p_bytes[0]; + block[1] = p_bytes[1]; + block[2] = p_bytes[2]; + block[3] = p_bytes[3]; + + block_len = pw_len; + } + else + { + block[0] = alt_result[0]; + block[1] = alt_result[1]; + block[2] = alt_result[2]; + block[3] = alt_result[3]; + block[4] = alt_result[4]; + block[5] = alt_result[5]; + block[6] = alt_result[6]; + block[7] = alt_result[7]; + + block_len = 32; + } + + if (j3) + { + block_len = memcat20 (block, block_len, s_bytes, salt_len); + } + + if (j7) + { + block_len = memcat20 (block, block_len, p_bytes, pw_len); + } + + if (j1) + { + block_len = memcat20 (block, block_len, &alt_result[0], 16); + block_len = memcat20_x80 (block, block_len, &alt_result[4], 16); + } + else + { + block_len = memcat20 (block, block_len, p_bytes_x80, pw_len); + } + + if (block_len >= 56) + { + sha256_transform (block, tmp); + + block[ 0] = block[16]; + block[ 1] = block[17]; + block[ 2] = block[18]; + block[ 3] = block[19]; + block[ 4] = 0; + block[ 5] = 0; + block[ 6] = 0; + block[ 7] = 0; + block[ 8] = 0; + block[ 9] = 0; + block[10] = 0; + block[11] = 0; + block[12] = 0; + block[13] = 0; + block[14] = 0; + block[15] = 0; + } + + block[15] = swap32 (block_len * 8); + + sha256_transform_no14 (block, tmp); + + bswap8 (tmp); + + alt_result[0] = tmp[0]; + alt_result[1] = tmp[1]; + alt_result[2] = tmp[2]; + alt_result[3] = tmp[3]; + alt_result[4] = tmp[4]; + alt_result[5] = tmp[5]; + alt_result[6] = tmp[6]; + alt_result[7] = tmp[7]; + } + + tmps[gid].alt_result[0] = alt_result[0]; + tmps[gid].alt_result[1] = alt_result[1]; + tmps[gid].alt_result[2] = alt_result[2]; + tmps[gid].alt_result[3] = alt_result[3]; + tmps[gid].alt_result[4] = alt_result[4]; + tmps[gid].alt_result[5] = alt_result[5]; + tmps[gid].alt_result[6] = alt_result[6]; + tmps[gid].alt_result[7] = alt_result[7]; +} + +__kernel void m07400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + const u32 r0 = tmps[gid].alt_result[0]; + const u32 r1 = tmps[gid].alt_result[1]; + const u32 r2 = tmps[gid].alt_result[2]; + const u32 r3 = tmps[gid].alt_result[3]; + + #define il_pos 0 + + #include COMPARE_M +} + +#else + +// this is basically a much cleaner version, but apparently drops speeds by over 100% :( + +#define PUTCHAR32_BE(a,p,c) ((u8 *)(a))[(p) ^ 3] = (u8) (c) +#define GETCHAR32_BE(a,p) ((u8 *)(a))[(p) ^ 3] + +typedef struct +{ + u32 state[8]; + u32 buf[32]; + int len; + +} sha256_ctx_t; + +void sha256_transform (const u32 w[16], u32 digest[8]) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + u32 e = digest[4]; + u32 f = digest[5]; + u32 g = digest[6]; + u32 h = digest[7]; + + u32 w0_t = w[ 0]; + u32 w1_t = w[ 1]; + u32 w2_t = w[ 2]; + u32 w3_t = w[ 3]; + u32 w4_t = w[ 4]; + u32 w5_t = w[ 5]; + u32 w6_t = w[ 6]; + u32 w7_t = w[ 7]; + u32 w8_t = w[ 8]; + u32 w9_t = w[ 9]; + u32 wa_t = w[10]; + u32 wb_t = w[11]; + u32 wc_t = w[12]; + u32 wd_t = w[13]; + u32 we_t = w[14]; + u32 wf_t = w[15]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void sha256_init (sha256_ctx_t *sha256_ctx) +{ + sha256_ctx->state[0] = SHA256M_A; + sha256_ctx->state[1] = SHA256M_B; + sha256_ctx->state[2] = SHA256M_C; + sha256_ctx->state[3] = SHA256M_D; + sha256_ctx->state[4] = SHA256M_E; + sha256_ctx->state[5] = SHA256M_F; + sha256_ctx->state[6] = SHA256M_G; + sha256_ctx->state[7] = SHA256M_H; + + sha256_ctx->len = 0; +} + +void sha256_update (sha256_ctx_t *sha256_ctx, const u32 *buf, int len) +{ + int pos = sha256_ctx->len & 0x3f; + + sha256_ctx->len += len; + + if ((pos + len) < 64) + { + for (int i = 0; i < len; i++) + { + PUTCHAR32_BE (sha256_ctx->buf, pos++, GETCHAR32_BE (buf, i)); + } + + return; + } + + int cnt = 64 - pos; + + for (int i = 0; i < cnt; i++) + { + PUTCHAR32_BE (sha256_ctx->buf, pos++, GETCHAR32_BE (buf, i)); + } + + sha256_transform (sha256_ctx->buf, sha256_ctx->state); + + len -= cnt; + + for (int i = 0; i < len; i++) + { + PUTCHAR32_BE (sha256_ctx->buf, i, GETCHAR32_BE (buf, cnt + i)); + } +} + +void sha256_final (sha256_ctx_t *sha256_ctx) +{ + int pos = sha256_ctx->len & 0x3f; + + for (int i = pos; i < 64; i++) + { + PUTCHAR32_BE (sha256_ctx->buf, i, 0); + } + + PUTCHAR32_BE (sha256_ctx->buf, pos, 0x80); + + if (pos >= 56) + { + sha256_transform (sha256_ctx->buf, sha256_ctx->state); + + sha256_ctx->buf[ 0] = 0; + sha256_ctx->buf[ 1] = 0; + sha256_ctx->buf[ 2] = 0; + sha256_ctx->buf[ 3] = 0; + sha256_ctx->buf[ 4] = 0; + sha256_ctx->buf[ 5] = 0; + sha256_ctx->buf[ 6] = 0; + sha256_ctx->buf[ 7] = 0; + sha256_ctx->buf[ 8] = 0; + sha256_ctx->buf[ 9] = 0; + sha256_ctx->buf[10] = 0; + sha256_ctx->buf[11] = 0; + sha256_ctx->buf[12] = 0; + sha256_ctx->buf[13] = 0; + sha256_ctx->buf[14] = 0; + sha256_ctx->buf[15] = 0; + } + + sha256_ctx->buf[15] = sha256_ctx->len * 8; + + sha256_transform (sha256_ctx->buf, sha256_ctx->state); +} + +__kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw[4]; + + pw[0] = swap32 (pws[gid].i[0]); + pw[1] = swap32 (pws[gid].i[1]); + pw[2] = swap32 (pws[gid].i[2]); + pw[3] = swap32 (pws[gid].i[3]); + + const u32 pw_len = pws[gid].pw_len; + + /** + * salt + */ + + u32 salt[4]; + + salt[0] = swap32 (salt_bufs[salt_pos].salt_buf[0]); + salt[1] = swap32 (salt_bufs[salt_pos].salt_buf[1]); + salt[2] = swap32 (salt_bufs[salt_pos].salt_buf[2]); + salt[3] = swap32 (salt_bufs[salt_pos].salt_buf[3]); + + u32 salt_len = salt_bufs[salt_pos].salt_len; + + /** + * begin + */ + + sha256_ctx_t sha256_ctx; + + sha256_init (&sha256_ctx); + + sha256_update (&sha256_ctx, pw, pw_len); + sha256_update (&sha256_ctx, salt, salt_len); + sha256_update (&sha256_ctx, pw, pw_len); + + sha256_final (&sha256_ctx); + + u32 tmp[8]; + + tmp[0] = sha256_ctx.state[0]; + tmp[1] = sha256_ctx.state[1]; + tmp[2] = sha256_ctx.state[2]; + tmp[3] = sha256_ctx.state[3]; + tmp[4] = sha256_ctx.state[4]; + tmp[5] = sha256_ctx.state[5]; + tmp[6] = sha256_ctx.state[6]; + tmp[7] = sha256_ctx.state[7]; + + sha256_init (&sha256_ctx); + + sha256_update (&sha256_ctx, pw, pw_len); + sha256_update (&sha256_ctx, salt, salt_len); + sha256_update (&sha256_ctx, tmp, pw_len); + + for (u32 j = pw_len; j; j >>= 1) + { + if (j & 1) + { + sha256_update (&sha256_ctx, tmp, 32); + } + else + { + sha256_update (&sha256_ctx, pw, pw_len); + } + } + + sha256_final (&sha256_ctx); + + tmps[gid].alt_result[0] = sha256_ctx.state[0]; + tmps[gid].alt_result[1] = sha256_ctx.state[1]; + tmps[gid].alt_result[2] = sha256_ctx.state[2]; + tmps[gid].alt_result[3] = sha256_ctx.state[3]; + tmps[gid].alt_result[4] = sha256_ctx.state[4]; + tmps[gid].alt_result[5] = sha256_ctx.state[5]; + tmps[gid].alt_result[6] = sha256_ctx.state[6]; + tmps[gid].alt_result[7] = sha256_ctx.state[7]; + + // p_bytes + + sha256_init (&sha256_ctx); + + for (u32 j = 0; j < pw_len; j++) + { + sha256_update (&sha256_ctx, pw, pw_len); + } + + sha256_final (&sha256_ctx); + + tmps[gid].p_bytes[0] = sha256_ctx.state[0]; + tmps[gid].p_bytes[1] = sha256_ctx.state[1]; + tmps[gid].p_bytes[2] = sha256_ctx.state[2]; + tmps[gid].p_bytes[3] = sha256_ctx.state[3]; + + // s_bytes + + sha256_init (&sha256_ctx); + + for (u32 j = 0; j < 16 + ((tmps[gid].alt_result[0] >> 24) & 0xff); j++) + { + sha256_update (&sha256_ctx, salt, salt_len); + } + + sha256_final (&sha256_ctx); + + tmps[gid].s_bytes[0] = sha256_ctx.state[0]; + tmps[gid].s_bytes[1] = sha256_ctx.state[1]; + tmps[gid].s_bytes[2] = sha256_ctx.state[2]; + tmps[gid].s_bytes[3] = sha256_ctx.state[3]; +} + +__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 p_bytes0[4]; + + p_bytes0[0] = tmps[gid].p_bytes[0]; + p_bytes0[1] = tmps[gid].p_bytes[1]; + p_bytes0[2] = tmps[gid].p_bytes[2]; + p_bytes0[3] = tmps[gid].p_bytes[3]; + + const u32 pw_len = pws[gid].pw_len; + + u32 s_bytes0[4]; + + s_bytes0[0] = tmps[gid].s_bytes[0]; + s_bytes0[1] = tmps[gid].s_bytes[1]; + s_bytes0[2] = tmps[gid].s_bytes[2]; + s_bytes0[3] = tmps[gid].s_bytes[3]; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + u32 wpc_len[8]; + + wpc_len[0] = 32 + 0 + 0 + pw_len; + wpc_len[1] = pw_len + 0 + 0 + 32; + wpc_len[2] = 32 + salt_len + 0 + pw_len; + wpc_len[3] = pw_len + salt_len + 0 + 32; + wpc_len[4] = 32 + 0 + pw_len + pw_len; + wpc_len[5] = pw_len + 0 + pw_len + 32; + wpc_len[6] = 32 + salt_len + pw_len + pw_len; + wpc_len[7] = pw_len + salt_len + pw_len + 32; + + u32 wpc[8][32] = { { 0 } }; + + for (u32 i = 0; i < 8; i++) + { + u32 block_len = 0; + + if (i & 1) + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); + } + } + else + { + block_len += 32; + } + + if (i & 2) + { + for (u32 j = 0; j < salt_len; j++) + { + PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (s_bytes0, j)); + } + } + + if (i & 4) + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); + } + } + + if (i & 1) + { + block_len += 32; + } + else + { + for (u32 j = 0; j < pw_len; j++) + { + PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); + } + } + + PUTCHAR32_BE (wpc[i], block_len, 0x80); + + if (block_len < 56) + { + wpc[i][15] = block_len * 8; + } + else + { + wpc[i][31] = block_len * 8; + } + } + + /** + * base + */ + + u32 alt_result[8]; + + alt_result[0] = tmps[gid].alt_result[0]; + alt_result[1] = tmps[gid].alt_result[1]; + alt_result[2] = tmps[gid].alt_result[2]; + alt_result[3] = tmps[gid].alt_result[3]; + alt_result[4] = tmps[gid].alt_result[4]; + alt_result[5] = tmps[gid].alt_result[5]; + alt_result[6] = tmps[gid].alt_result[6]; + alt_result[7] = tmps[gid].alt_result[7]; + + /* Repeatedly run the collected hash value through SHA256 to burn + CPU cycles. */ + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + const u32 j1 = (j & 1) ? 1 : 0; + const u32 j3 = (j % 3) ? 2 : 0; + const u32 j7 = (j % 7) ? 4 : 0; + + const u32 pc = j1 + j3 + j7; + + u32 block[32]; + + block[ 0] = wpc[pc][ 0]; + block[ 1] = wpc[pc][ 1]; + block[ 2] = wpc[pc][ 2]; + block[ 3] = wpc[pc][ 3]; + block[ 4] = wpc[pc][ 4]; + block[ 5] = wpc[pc][ 5]; + block[ 6] = wpc[pc][ 6]; + block[ 7] = wpc[pc][ 7]; + block[ 8] = wpc[pc][ 8]; + block[ 9] = wpc[pc][ 9]; + block[10] = wpc[pc][10]; + block[11] = wpc[pc][11]; + block[12] = wpc[pc][12]; + block[13] = wpc[pc][13]; + block[14] = wpc[pc][14]; + block[15] = wpc[pc][15]; + block[16] = wpc[pc][16]; + block[17] = wpc[pc][17]; + block[18] = wpc[pc][18]; + block[19] = wpc[pc][19]; + block[20] = wpc[pc][20]; + block[21] = wpc[pc][21]; + block[22] = wpc[pc][22]; + block[23] = wpc[pc][23]; + block[24] = wpc[pc][24]; + block[25] = wpc[pc][25]; + block[26] = wpc[pc][26]; + block[27] = wpc[pc][27]; + block[28] = wpc[pc][28]; + block[29] = wpc[pc][29]; + block[30] = wpc[pc][30]; + block[31] = wpc[pc][31]; + + const u32 block_len = wpc_len[pc]; + + if (j1) + { + #ifdef _unroll + #pragma unroll + #endif + for (u32 k = 0, p = block_len - 32; k < 32; k++, p++) + { + PUTCHAR32_BE (block, p, GETCHAR32_BE (alt_result, k)); + } + } + else + { + block[0] = alt_result[0]; + block[1] = alt_result[1]; + block[2] = alt_result[2]; + block[3] = alt_result[3]; + block[4] = alt_result[4]; + block[5] = alt_result[5]; + block[6] = alt_result[6]; + block[7] = alt_result[7]; + } + + alt_result[0] = SHA256M_A; + alt_result[1] = SHA256M_B; + alt_result[2] = SHA256M_C; + alt_result[3] = SHA256M_D; + alt_result[4] = SHA256M_E; + alt_result[5] = SHA256M_F; + alt_result[6] = SHA256M_G; + alt_result[7] = SHA256M_H; + + sha256_transform (block, alt_result); + + if (block_len >= 56) + { + sha256_transform (block + 16, alt_result); + } + } + + tmps[gid].alt_result[0] = alt_result[0]; + tmps[gid].alt_result[1] = alt_result[1]; + tmps[gid].alt_result[2] = alt_result[2]; + tmps[gid].alt_result[3] = alt_result[3]; + tmps[gid].alt_result[4] = alt_result[4]; + tmps[gid].alt_result[5] = alt_result[5]; + tmps[gid].alt_result[6] = alt_result[6]; + tmps[gid].alt_result[7] = alt_result[7]; +} + +__kernel void m07400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + const u32 r0 = swap32 (tmps[gid].alt_result[0]); + const u32 r1 = swap32 (tmps[gid].alt_result[1]); + const u32 r2 = swap32 (tmps[gid].alt_result[2]); + const u32 r3 = swap32 (tmps[gid].alt_result[3]); + + #define il_pos 0 + + #include COMPARE_M +} + +#endif diff --git a/OpenCL/m07400-pure.cl b/OpenCL/m07400-pure.cl deleted file mode 100644 index 936add9f0..000000000 --- a/OpenCL/m07400-pure.cl +++ /dev/null @@ -1,365 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_sha256.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -__kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * init - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (w[idx]); - } - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - const u32 salt_lenv = ceil ((float) salt_len / 4); - - u32 s[64] = { 0 }; - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - for (int idx = 0; idx < salt_lenv; idx++) - { - s[idx] = swap32_S (s[idx]); - } - - /** - * prepare - */ - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - sha256_update (&ctx, w, pw_len); - - sha256_final (&ctx); - - u32 final[16] = { 0 }; - - final[0] = ctx.h[0]; - final[1] = ctx.h[1]; - final[2] = ctx.h[2]; - final[3] = ctx.h[3]; - final[4] = ctx.h[4]; - final[5] = ctx.h[5]; - final[6] = ctx.h[6]; - final[7] = ctx.h[7]; - - // alt_result - - sha256_init (&ctx); - - sha256_update (&ctx, w, pw_len); - - sha256_update (&ctx, s, salt_len); - - int pl; - - for (pl = pw_len; pl > 32; pl -= 32) - { - sha256_update (&ctx, final, 32); - } - - u32 t_final[16] = { 0 }; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 8; i++) t_final[i] = final[i]; - - truncate_block_16x4_be (t_final + 0, t_final + 4, t_final + 8, t_final + 12, pl); - - sha256_update (&ctx, t_final, pl); - - for (int cnt = pw_len; cnt > 0; cnt >>= 1) - { - if ((cnt & 1) != 0) - { - sha256_update (&ctx, final, 32); - } - else - { - sha256_update (&ctx, w, pw_len); - } - } - - sha256_final (&ctx); - - tmps[gid].alt_result[0] = ctx.h[0]; - tmps[gid].alt_result[1] = ctx.h[1]; - tmps[gid].alt_result[2] = ctx.h[2]; - tmps[gid].alt_result[3] = ctx.h[3]; - tmps[gid].alt_result[4] = ctx.h[4]; - tmps[gid].alt_result[5] = ctx.h[5]; - tmps[gid].alt_result[6] = ctx.h[6]; - tmps[gid].alt_result[7] = ctx.h[7]; - - // p_bytes - - sha256_init (&ctx); - - for (u32 j = 0; j < pw_len; j++) - { - sha256_update (&ctx, w, pw_len); - } - - sha256_final (&ctx); - - final[ 0] = ctx.h[0]; - final[ 1] = ctx.h[1]; - final[ 2] = ctx.h[2]; - final[ 3] = ctx.h[3]; - final[ 4] = ctx.h[4]; - final[ 5] = ctx.h[5]; - final[ 6] = ctx.h[6]; - final[ 7] = ctx.h[7]; - final[ 8] = 0; - final[ 9] = 0; - final[10] = 0; - final[11] = 0; - final[12] = 0; - final[13] = 0; - final[14] = 0; - final[15] = 0; - - u32 p_final[64] = { 0 }; - - int idx; - - for (pl = pw_len, idx = 0; pl > 32; pl -= 32, idx += 8) - { - p_final[idx + 0] = final[0]; - p_final[idx + 1] = final[1]; - p_final[idx + 2] = final[2]; - p_final[idx + 3] = final[3]; - p_final[idx + 4] = final[4]; - p_final[idx + 5] = final[5]; - p_final[idx + 6] = final[6]; - p_final[idx + 7] = final[7]; - } - - truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); - - p_final[idx + 0] = final[0]; - p_final[idx + 1] = final[1]; - p_final[idx + 2] = final[2]; - p_final[idx + 3] = final[3]; - p_final[idx + 4] = final[4]; - p_final[idx + 5] = final[5]; - p_final[idx + 6] = final[6]; - p_final[idx + 7] = final[7]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) tmps[gid].p_bytes[i] = p_final[i]; - - // s_bytes - - sha256_init (&ctx); - - for (u32 j = 0; j < 16 + (tmps[gid].alt_result[0] >> 24); j++) - { - sha256_update (&ctx, s, salt_len); - } - - sha256_final (&ctx); - - final[ 0] = ctx.h[0]; - final[ 1] = ctx.h[1]; - final[ 2] = ctx.h[2]; - final[ 3] = ctx.h[3]; - final[ 4] = ctx.h[4]; - final[ 5] = ctx.h[5]; - final[ 6] = ctx.h[6]; - final[ 7] = ctx.h[7]; - final[ 8] = 0; - final[ 9] = 0; - final[10] = 0; - final[11] = 0; - final[12] = 0; - final[13] = 0; - final[14] = 0; - final[15] = 0; - - u32 s_final[64] = { 0 }; - - for (pl = salt_len, idx = 0; pl > 32; pl -= 32, idx += 8) - { - s_final[idx + 0] = final[0]; - s_final[idx + 1] = final[1]; - s_final[idx + 2] = final[2]; - s_final[idx + 3] = final[3]; - s_final[idx + 4] = final[4]; - s_final[idx + 5] = final[5]; - s_final[idx + 6] = final[6]; - s_final[idx + 7] = final[7]; - } - - truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); - - s_final[idx + 0] = final[0]; - s_final[idx + 1] = final[1]; - s_final[idx + 2] = final[2]; - s_final[idx + 3] = final[3]; - s_final[idx + 4] = final[4]; - s_final[idx + 5] = final[5]; - s_final[idx + 6] = final[6]; - s_final[idx + 7] = final[7]; - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 0; i < 64; i++) tmps[gid].s_bytes[i] = s_final[i]; -} - -__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 pw_len = pws[gid].pw_len; - - const u32 salt_len = salt_bufs[salt_pos].salt_len; - - u32 alt_result[16] = { 0 }; - - alt_result[0] = tmps[gid].alt_result[0]; - alt_result[1] = tmps[gid].alt_result[1]; - alt_result[2] = tmps[gid].alt_result[2]; - alt_result[3] = tmps[gid].alt_result[3]; - alt_result[4] = tmps[gid].alt_result[4]; - alt_result[5] = tmps[gid].alt_result[5]; - alt_result[6] = tmps[gid].alt_result[6]; - alt_result[7] = tmps[gid].alt_result[7]; - - /* Repeatedly run the collected hash value through sha256 to burn - CPU cycles. */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - sha256_ctx_t ctx; - - sha256_init (&ctx); - - if (j & 1) - { - sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - else - { - sha256_update (&ctx, alt_result, 32); - } - - if (j % 3) - { - sha256_update_global (&ctx, tmps[gid].s_bytes, salt_len); - } - - if (j % 7) - { - sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - - if (j & 1) - { - sha256_update (&ctx, alt_result, 32); - } - else - { - sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); - } - - sha256_final (&ctx); - - alt_result[0] = ctx.h[0]; - alt_result[1] = ctx.h[1]; - alt_result[2] = ctx.h[2]; - alt_result[3] = ctx.h[3]; - alt_result[4] = ctx.h[4]; - alt_result[5] = ctx.h[5]; - alt_result[6] = ctx.h[6]; - alt_result[7] = ctx.h[7]; - } - - tmps[gid].alt_result[0] = alt_result[0]; - tmps[gid].alt_result[1] = alt_result[1]; - tmps[gid].alt_result[2] = alt_result[2]; - tmps[gid].alt_result[3] = alt_result[3]; - tmps[gid].alt_result[4] = alt_result[4]; - tmps[gid].alt_result[5] = alt_result[5]; - tmps[gid].alt_result[6] = alt_result[6]; - tmps[gid].alt_result[7] = alt_result[7]; -} - -__kernel void m07400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - const u32 r0 = swap32_S (tmps[gid].alt_result[0]); - const u32 r1 = swap32_S (tmps[gid].alt_result[1]); - const u32 r2 = swap32_S (tmps[gid].alt_result[2]); - const u32 r3 = swap32_S (tmps[gid].alt_result[3]); - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m07400.cl b/OpenCL/m07400.cl index e3a9fbda5..936add9f0 100644 --- a/OpenCL/m07400.cl +++ b/OpenCL/m07400.cl @@ -8,778 +8,11 @@ #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" +#include "inc_hash_sha256.cl" #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" -__constant u32a k_sha256[64] = -{ - SHA256C00, SHA256C01, SHA256C02, SHA256C03, - SHA256C04, SHA256C05, SHA256C06, SHA256C07, - SHA256C08, SHA256C09, SHA256C0a, SHA256C0b, - SHA256C0c, SHA256C0d, SHA256C0e, SHA256C0f, - SHA256C10, SHA256C11, SHA256C12, SHA256C13, - SHA256C14, SHA256C15, SHA256C16, SHA256C17, - SHA256C18, SHA256C19, SHA256C1a, SHA256C1b, - SHA256C1c, SHA256C1d, SHA256C1e, SHA256C1f, - SHA256C20, SHA256C21, SHA256C22, SHA256C23, - SHA256C24, SHA256C25, SHA256C26, SHA256C27, - SHA256C28, SHA256C29, SHA256C2a, SHA256C2b, - SHA256C2c, SHA256C2d, SHA256C2e, SHA256C2f, - SHA256C30, SHA256C31, SHA256C32, SHA256C33, - SHA256C34, SHA256C35, SHA256C36, SHA256C37, - SHA256C38, SHA256C39, SHA256C3a, SHA256C3b, - SHA256C3c, SHA256C3d, SHA256C3e, SHA256C3f, -}; - -#if 1 - -void sha256_transform (const u32 w[16], u32 digest[8]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - u32 e = digest[4]; - u32 f = digest[5]; - u32 g = digest[6]; - u32 h = digest[7]; - - u32 w0_t = swap32 (w[ 0]); - u32 w1_t = swap32 (w[ 1]); - u32 w2_t = swap32 (w[ 2]); - u32 w3_t = swap32 (w[ 3]); - u32 w4_t = swap32 (w[ 4]); - u32 w5_t = swap32 (w[ 5]); - u32 w6_t = swap32 (w[ 6]); - u32 w7_t = swap32 (w[ 7]); - u32 w8_t = swap32 (w[ 8]); - u32 w9_t = swap32 (w[ 9]); - u32 wa_t = swap32 (w[10]); - u32 wb_t = swap32 (w[11]); - u32 wc_t = swap32 (w[12]); - u32 wd_t = swap32 (w[13]); - u32 we_t = swap32 (w[14]); - u32 wf_t = swap32 (w[15]); - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void sha256_transform_no14 (const u32 w[16], u32 digest[8]) -{ - u32 w_t[16]; - - w_t[ 0] = w[ 0]; - w_t[ 1] = w[ 1]; - w_t[ 2] = w[ 2]; - w_t[ 3] = w[ 3]; - w_t[ 4] = w[ 4]; - w_t[ 5] = w[ 5]; - w_t[ 6] = w[ 6]; - w_t[ 7] = w[ 7]; - w_t[ 8] = w[ 8]; - w_t[ 9] = w[ 9]; - w_t[10] = w[10]; - w_t[11] = w[11]; - w_t[12] = w[12]; - w_t[13] = w[13]; - w_t[14] = 0; - w_t[15] = w[15]; - - sha256_transform (w_t, digest); -} - -void init_ctx (u32 digest[8]) -{ - digest[0] = SHA256M_A; - digest[1] = SHA256M_B; - digest[2] = SHA256M_C; - digest[3] = SHA256M_D; - digest[4] = SHA256M_E; - digest[5] = SHA256M_F; - digest[6] = SHA256M_G; - digest[7] = SHA256M_H; -} - -void bzero16 (u32 block[16]) -{ - block[ 0] = 0; - block[ 1] = 0; - block[ 2] = 0; - block[ 3] = 0; - block[ 4] = 0; - block[ 5] = 0; - block[ 6] = 0; - block[ 7] = 0; - block[ 8] = 0; - block[ 9] = 0; - block[10] = 0; - block[11] = 0; - block[12] = 0; - block[13] = 0; - block[14] = 0; - block[15] = 0; -} - -void bswap8 (u32 block[16]) -{ - block[ 0] = swap32 (block[ 0]); - block[ 1] = swap32 (block[ 1]); - block[ 2] = swap32 (block[ 2]); - block[ 3] = swap32 (block[ 3]); - block[ 4] = swap32 (block[ 4]); - block[ 5] = swap32 (block[ 5]); - block[ 6] = swap32 (block[ 6]); - block[ 7] = swap32 (block[ 7]); -} - -u32 memcat16 (u32 block[16], const u32 block_len, const u32 append[4], const u32 append_len) -{ - const u32 mod = block_len & 3; - const u32 div = block_len / 4; - - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - const int offset_minus_4 = 4 - block_len; - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - #endif - - switch (div) - { - case 0: block[ 0] |= tmp0; - block[ 1] = tmp1; - block[ 2] = tmp2; - block[ 3] = tmp3; - block[ 4] = tmp4; - break; - case 1: block[ 1] |= tmp0; - block[ 2] = tmp1; - block[ 3] = tmp2; - block[ 4] = tmp3; - block[ 5] = tmp4; - break; - case 2: block[ 2] |= tmp0; - block[ 3] = tmp1; - block[ 4] = tmp2; - block[ 5] = tmp3; - block[ 6] = tmp4; - break; - case 3: block[ 3] |= tmp0; - block[ 4] = tmp1; - block[ 5] = tmp2; - block[ 6] = tmp3; - block[ 7] = tmp4; - break; - case 4: block[ 4] |= tmp0; - block[ 5] = tmp1; - block[ 6] = tmp2; - block[ 7] = tmp3; - block[ 8] = tmp4; - break; - case 5: block[ 5] |= tmp0; - block[ 6] = tmp1; - block[ 7] = tmp2; - block[ 8] = tmp3; - block[ 9] = tmp4; - break; - case 6: block[ 6] |= tmp0; - block[ 7] = tmp1; - block[ 8] = tmp2; - block[ 9] = tmp3; - block[10] = tmp4; - break; - case 7: block[ 7] |= tmp0; - block[ 8] = tmp1; - block[ 9] = tmp2; - block[10] = tmp3; - block[11] = tmp4; - break; - case 8: block[ 8] |= tmp0; - block[ 9] = tmp1; - block[10] = tmp2; - block[11] = tmp3; - block[12] = tmp4; - break; - case 9: block[ 9] |= tmp0; - block[10] = tmp1; - block[11] = tmp2; - block[12] = tmp3; - block[13] = tmp4; - break; - case 10: block[10] |= tmp0; - block[11] = tmp1; - block[12] = tmp2; - block[13] = tmp3; - block[14] = tmp4; - break; - case 11: block[11] |= tmp0; - block[12] = tmp1; - block[13] = tmp2; - block[14] = tmp3; - block[15] = tmp4; - break; - case 12: block[12] |= tmp0; - block[13] = tmp1; - block[14] = tmp2; - block[15] = tmp3; - break; - case 13: block[13] |= tmp0; - block[14] = tmp1; - block[15] = tmp2; - break; - case 14: block[14] |= tmp0; - block[15] = tmp1; - break; - case 15: block[15] |= tmp0; - break; - } - - u32 new_len = block_len + append_len; - - return new_len; -} - -u32 memcat16c (u32 block[16], const u32 block_len, const u32 append[4], const u32 append_len, u32 digest[8]) -{ - const u32 mod = block_len & 3; - const u32 div = block_len / 4; - - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - const int offset_minus_4 = 4 - block_len; - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - #endif - - u32 carry[4] = { 0, 0, 0, 0 }; - - switch (div) - { - case 0: block[ 0] |= tmp0; - block[ 1] = tmp1; - block[ 2] = tmp2; - block[ 3] = tmp3; - block[ 4] = tmp4; - break; - case 1: block[ 1] |= tmp0; - block[ 2] = tmp1; - block[ 3] = tmp2; - block[ 4] = tmp3; - block[ 5] = tmp4; - break; - case 2: block[ 2] |= tmp0; - block[ 3] = tmp1; - block[ 4] = tmp2; - block[ 5] = tmp3; - block[ 6] = tmp4; - break; - case 3: block[ 3] |= tmp0; - block[ 4] = tmp1; - block[ 5] = tmp2; - block[ 6] = tmp3; - block[ 7] = tmp4; - break; - case 4: block[ 4] |= tmp0; - block[ 5] = tmp1; - block[ 6] = tmp2; - block[ 7] = tmp3; - block[ 8] = tmp4; - break; - case 5: block[ 5] |= tmp0; - block[ 6] = tmp1; - block[ 7] = tmp2; - block[ 8] = tmp3; - block[ 9] = tmp4; - break; - case 6: block[ 6] |= tmp0; - block[ 7] = tmp1; - block[ 8] = tmp2; - block[ 9] = tmp3; - block[10] = tmp4; - break; - case 7: block[ 7] |= tmp0; - block[ 8] = tmp1; - block[ 9] = tmp2; - block[10] = tmp3; - block[11] = tmp4; - break; - case 8: block[ 8] |= tmp0; - block[ 9] = tmp1; - block[10] = tmp2; - block[11] = tmp3; - block[12] = tmp4; - break; - case 9: block[ 9] |= tmp0; - block[10] = tmp1; - block[11] = tmp2; - block[12] = tmp3; - block[13] = tmp4; - break; - case 10: block[10] |= tmp0; - block[11] = tmp1; - block[12] = tmp2; - block[13] = tmp3; - block[14] = tmp4; - break; - case 11: block[11] |= tmp0; - block[12] = tmp1; - block[13] = tmp2; - block[14] = tmp3; - block[15] = tmp4; - break; - case 12: block[12] |= tmp0; - block[13] = tmp1; - block[14] = tmp2; - block[15] = tmp3; - carry[ 0] = tmp4; - break; - case 13: block[13] |= tmp0; - block[14] = tmp1; - block[15] = tmp2; - carry[ 0] = tmp3; - carry[ 1] = tmp4; - break; - case 14: block[14] |= tmp0; - block[15] = tmp1; - carry[ 0] = tmp2; - carry[ 1] = tmp3; - carry[ 2] = tmp4; - break; - case 15: block[15] |= tmp0; - carry[ 0] = tmp1; - carry[ 1] = tmp2; - carry[ 2] = tmp3; - carry[ 3] = tmp4; - break; - } - - u32 new_len = block_len + append_len; - - if (new_len >= 64) - { - new_len -= 64; - - sha256_transform (block, digest); - - bzero16 (block); - - block[0] = carry[0]; - block[1] = carry[1]; - block[2] = carry[2]; - block[3] = carry[3]; - } - - return new_len; -} - -u32 memcat20 (u32 block[20], const u32 block_len, const u32 append[4], const u32 append_len) -{ - const u32 mod = block_len & 3; - const u32 div = block_len / 4; - - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - const int offset_minus_4 = 4 - block_len; - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0, append[3], offset_minus_4); - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0, selector); - #endif - - switch (div) - { - case 0: block[ 0] |= tmp0; - block[ 1] = tmp1; - block[ 2] = tmp2; - block[ 3] = tmp3; - block[ 4] = tmp4; - break; - case 1: block[ 1] |= tmp0; - block[ 2] = tmp1; - block[ 3] = tmp2; - block[ 4] = tmp3; - block[ 5] = tmp4; - break; - case 2: block[ 2] |= tmp0; - block[ 3] = tmp1; - block[ 4] = tmp2; - block[ 5] = tmp3; - block[ 6] = tmp4; - break; - case 3: block[ 3] |= tmp0; - block[ 4] = tmp1; - block[ 5] = tmp2; - block[ 6] = tmp3; - block[ 7] = tmp4; - break; - case 4: block[ 4] |= tmp0; - block[ 5] = tmp1; - block[ 6] = tmp2; - block[ 7] = tmp3; - block[ 8] = tmp4; - break; - case 5: block[ 5] |= tmp0; - block[ 6] = tmp1; - block[ 7] = tmp2; - block[ 8] = tmp3; - block[ 9] = tmp4; - break; - case 6: block[ 6] |= tmp0; - block[ 7] = tmp1; - block[ 8] = tmp2; - block[ 9] = tmp3; - block[10] = tmp4; - break; - case 7: block[ 7] |= tmp0; - block[ 8] = tmp1; - block[ 9] = tmp2; - block[10] = tmp3; - block[11] = tmp4; - break; - case 8: block[ 8] |= tmp0; - block[ 9] = tmp1; - block[10] = tmp2; - block[11] = tmp3; - block[12] = tmp4; - break; - case 9: block[ 9] |= tmp0; - block[10] = tmp1; - block[11] = tmp2; - block[12] = tmp3; - block[13] = tmp4; - break; - case 10: block[10] |= tmp0; - block[11] = tmp1; - block[12] = tmp2; - block[13] = tmp3; - block[14] = tmp4; - break; - case 11: block[11] |= tmp0; - block[12] = tmp1; - block[13] = tmp2; - block[14] = tmp3; - block[15] = tmp4; - break; - case 12: block[12] |= tmp0; - block[13] = tmp1; - block[14] = tmp2; - block[15] = tmp3; - block[16] = tmp4; - break; - case 13: block[13] |= tmp0; - block[14] = tmp1; - block[15] = tmp2; - block[16] = tmp3; - block[17] = tmp4; - break; - case 14: block[14] |= tmp0; - block[15] = tmp1; - block[16] = tmp2; - block[17] = tmp3; - block[18] = tmp4; - break; - case 15: block[15] |= tmp0; - block[16] = tmp1; - block[17] = tmp2; - block[18] = tmp3; - block[19] = tmp4; - break; - } - - return block_len + append_len; -} - -u32 memcat20_x80 (u32 block[20], const u32 block_len, const u32 append[4], const u32 append_len) -{ - const u32 mod = block_len & 3; - const u32 div = block_len / 4; - - u32 tmp0; - u32 tmp1; - u32 tmp2; - u32 tmp3; - u32 tmp4; - - #if defined IS_AMD || defined IS_GENERIC - const int offset_minus_4 = 4 - block_len; - - tmp0 = amd_bytealign (append[0], 0, offset_minus_4); - tmp1 = amd_bytealign (append[1], append[0], offset_minus_4); - tmp2 = amd_bytealign (append[2], append[1], offset_minus_4); - tmp3 = amd_bytealign (append[3], append[2], offset_minus_4); - tmp4 = amd_bytealign ( 0x80, append[3], offset_minus_4); - - if (mod == 0) - { - tmp0 = tmp1; - tmp1 = tmp2; - tmp2 = tmp3; - tmp3 = tmp4; - tmp4 = 0x80; - } - #endif - - #ifdef IS_NV - const int offset_minus_4 = 4 - (block_len & 3); - - const int selector = (0x76543210 >> (offset_minus_4 * 4)) & 0xffff; - - tmp0 = __byte_perm ( 0, append[0], selector); - tmp1 = __byte_perm (append[0], append[1], selector); - tmp2 = __byte_perm (append[1], append[2], selector); - tmp3 = __byte_perm (append[2], append[3], selector); - tmp4 = __byte_perm (append[3], 0x80, selector); - #endif - - switch (div) - { - case 0: block[ 0] |= tmp0; - block[ 1] = tmp1; - block[ 2] = tmp2; - block[ 3] = tmp3; - block[ 4] = tmp4; - break; - case 1: block[ 1] |= tmp0; - block[ 2] = tmp1; - block[ 3] = tmp2; - block[ 4] = tmp3; - block[ 5] = tmp4; - break; - case 2: block[ 2] |= tmp0; - block[ 3] = tmp1; - block[ 4] = tmp2; - block[ 5] = tmp3; - block[ 6] = tmp4; - break; - case 3: block[ 3] |= tmp0; - block[ 4] = tmp1; - block[ 5] = tmp2; - block[ 6] = tmp3; - block[ 7] = tmp4; - break; - case 4: block[ 4] |= tmp0; - block[ 5] = tmp1; - block[ 6] = tmp2; - block[ 7] = tmp3; - block[ 8] = tmp4; - break; - case 5: block[ 5] |= tmp0; - block[ 6] = tmp1; - block[ 7] = tmp2; - block[ 8] = tmp3; - block[ 9] = tmp4; - break; - case 6: block[ 6] |= tmp0; - block[ 7] = tmp1; - block[ 8] = tmp2; - block[ 9] = tmp3; - block[10] = tmp4; - break; - case 7: block[ 7] |= tmp0; - block[ 8] = tmp1; - block[ 9] = tmp2; - block[10] = tmp3; - block[11] = tmp4; - break; - case 8: block[ 8] |= tmp0; - block[ 9] = tmp1; - block[10] = tmp2; - block[11] = tmp3; - block[12] = tmp4; - break; - case 9: block[ 9] |= tmp0; - block[10] = tmp1; - block[11] = tmp2; - block[12] = tmp3; - block[13] = tmp4; - break; - case 10: block[10] |= tmp0; - block[11] = tmp1; - block[12] = tmp2; - block[13] = tmp3; - block[14] = tmp4; - break; - case 11: block[11] |= tmp0; - block[12] = tmp1; - block[13] = tmp2; - block[14] = tmp3; - block[15] = tmp4; - break; - case 12: block[12] |= tmp0; - block[13] = tmp1; - block[14] = tmp2; - block[15] = tmp3; - block[16] = tmp4; - break; - case 13: block[13] |= tmp0; - block[14] = tmp1; - block[15] = tmp2; - block[16] = tmp3; - block[17] = tmp4; - break; - case 14: block[14] |= tmp0; - block[15] = tmp1; - block[16] = tmp2; - block[17] = tmp3; - block[18] = tmp4; - break; - case 15: block[15] |= tmp0; - block[16] = tmp1; - block[17] = tmp2; - block[18] = tmp3; - block[19] = tmp4; - break; - } - - return block_len + append_len; -} - __kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** @@ -790,839 +23,256 @@ __kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rul if (gid >= gid_max) return; - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; + /** + * init + */ const u32 pw_len = pws[gid].pw_len; - /** - * salt - */ + const u32 pw_lenv = ceil ((float) pw_len / 4); - u32 salt_buf[4]; + u32 w[64] = { 0 }; - salt_buf[0] = salt_bufs[salt_pos].salt_buf[0]; - salt_buf[1] = salt_bufs[salt_pos].salt_buf[1]; - salt_buf[2] = salt_bufs[salt_pos].salt_buf[2]; - salt_buf[3] = salt_bufs[salt_pos].salt_buf[3]; - - u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * buffers - */ - - u32 block_len; // never reaches > 64 - u32 transform_len; // required for w[15] = len * 8 - - u32 block[16]; - - u32 alt_result[8]; - u32 p_bytes[8]; - u32 s_bytes[8]; - - /* Prepare for the real work. */ - - block_len = 0; - - bzero16 (block); - - /* Add key. */ - - block_len = memcat16 (block, block_len, w0, pw_len); - - /* Add salt. */ - - block_len = memcat16 (block, block_len, salt_buf, salt_len); - - /* Add key again. */ - - block_len = memcat16 (block, block_len, w0, pw_len); - - append_0x80_1x16 (block, block_len); - - block[15] = swap32 (block_len * 8); - - init_ctx (alt_result); - - sha256_transform (block, alt_result); - - bswap8 (alt_result); - - block_len = 0; - - bzero16 (block); - - u32 alt_result_tmp[8]; - - alt_result_tmp[0] = alt_result[0]; - alt_result_tmp[1] = alt_result[1]; - alt_result_tmp[2] = alt_result[2]; - alt_result_tmp[3] = alt_result[3]; - alt_result_tmp[4] = 0; - alt_result_tmp[5] = 0; - alt_result_tmp[6] = 0; - alt_result_tmp[7] = 0; - - truncate_block_4x4_le (alt_result_tmp, pw_len); - - /* Add the key string. */ - - block_len = memcat16 (block, block_len, w0, pw_len); - - /* The last part is the salt string. This must be at most 8 - characters and it ends at the first `$' character (for - compatibility with existing implementations). */ - - block_len = memcat16 (block, block_len, salt_buf, salt_len); - - /* Now get result of this (32 bytes) and add it to the other - context. */ - - block_len = memcat16 (block, block_len, alt_result_tmp, pw_len); - - transform_len = block_len; - - /* Take the binary representation of the length of the key and for every - 1 add the alternate sum, for every 0 the key. */ - - alt_result_tmp[0] = alt_result[0]; - alt_result_tmp[1] = alt_result[1]; - alt_result_tmp[2] = alt_result[2]; - alt_result_tmp[3] = alt_result[3]; - alt_result_tmp[4] = alt_result[4]; - alt_result_tmp[5] = alt_result[5]; - alt_result_tmp[6] = alt_result[6]; - alt_result_tmp[7] = alt_result[7]; - - init_ctx (alt_result); - - for (u32 j = pw_len; j; j >>= 1) + for (int idx = 0; idx < pw_lenv; idx++) { - if (j & 1) - { - block_len = memcat16c (block, block_len, &alt_result_tmp[0], 16, alt_result); - block_len = memcat16c (block, block_len, &alt_result_tmp[4], 16, alt_result); + w[idx] = pws[gid].i[idx]; - transform_len += 32; - } - else - { - block_len = memcat16c (block, block_len, w0, pw_len, alt_result); - - transform_len += pw_len; - } + barrier (CLK_GLOBAL_MEM_FENCE); } - append_0x80_1x16 (block, block_len); - - if (block_len >= 56) + for (int idx = 0; idx < pw_lenv; idx++) { - sha256_transform (block, alt_result); - - bzero16 (block); + w[idx] = swap32_S (w[idx]); } - block[15] = swap32 (transform_len * 8); - - sha256_transform (block, alt_result); - - bswap8 (alt_result); - - tmps[gid].alt_result[0] = alt_result[0]; - tmps[gid].alt_result[1] = alt_result[1]; - tmps[gid].alt_result[2] = alt_result[2]; - tmps[gid].alt_result[3] = alt_result[3]; - tmps[gid].alt_result[4] = alt_result[4]; - tmps[gid].alt_result[5] = alt_result[5]; - tmps[gid].alt_result[6] = alt_result[6]; - tmps[gid].alt_result[7] = alt_result[7]; - - /* Start computation of P byte sequence. */ - - block_len = 0; - - transform_len = 0; - - bzero16 (block); - - /* For every character in the password add the entire password. */ - - init_ctx (p_bytes); - - for (u32 j = 0; j < pw_len; j++) - { - block_len = memcat16c (block, block_len, w0, pw_len, p_bytes); - - transform_len += pw_len; - } - - /* Finish the digest. */ - - append_0x80_1x16 (block, block_len); - - if (block_len >= 56) - { - sha256_transform (block, p_bytes); - - bzero16 (block); - } - - block[15] = swap32 (transform_len * 8); - - sha256_transform (block, p_bytes); - - bswap8 (p_bytes); - - truncate_block_4x4_le (p_bytes, pw_len); - - tmps[gid].p_bytes[0] = p_bytes[0]; - tmps[gid].p_bytes[1] = p_bytes[1]; - tmps[gid].p_bytes[2] = p_bytes[2]; - tmps[gid].p_bytes[3] = p_bytes[3]; - - /* Start computation of S byte sequence. */ - - block_len = 0; - - transform_len = 0; - - bzero16 (block); - - /* For every character in the password add the entire password. */ - - init_ctx (s_bytes); - - for (u32 j = 0; j < 16 + (alt_result[0] & 0xff); j++) - { - block_len = memcat16c (block, block_len, salt_buf, salt_len, s_bytes); - - transform_len += salt_len; - } - - /* Finish the digest. */ - - append_0x80_1x16 (block, block_len); - - if (block_len >= 56) - { - sha256_transform (block, s_bytes); - - bzero16 (block); - } - - block[15] = swap32 (transform_len * 8); - - sha256_transform (block, s_bytes); - - bswap8 (s_bytes); - - truncate_block_4x4_le (s_bytes, salt_len); - - tmps[gid].s_bytes[0] = s_bytes[0]; - tmps[gid].s_bytes[1] = s_bytes[1]; - tmps[gid].s_bytes[2] = s_bytes[2]; - tmps[gid].s_bytes[3] = s_bytes[3]; -} - -__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 pw_len = pws[gid].pw_len; - - /** - * base - */ - - u32 p_bytes[4]; - - p_bytes[0] = tmps[gid].p_bytes[0]; - p_bytes[1] = tmps[gid].p_bytes[1]; - p_bytes[2] = tmps[gid].p_bytes[2]; - p_bytes[3] = tmps[gid].p_bytes[3]; - - u32 p_bytes_x80[4]; - - p_bytes_x80[0] = tmps[gid].p_bytes[0]; - p_bytes_x80[1] = tmps[gid].p_bytes[1]; - p_bytes_x80[2] = tmps[gid].p_bytes[2]; - p_bytes_x80[3] = tmps[gid].p_bytes[3]; - - append_0x80_1x4 (p_bytes_x80, pw_len); - - u32 s_bytes[4]; - - s_bytes[0] = tmps[gid].s_bytes[0]; - s_bytes[1] = tmps[gid].s_bytes[1]; - s_bytes[2] = tmps[gid].s_bytes[2]; - s_bytes[3] = tmps[gid].s_bytes[3]; - - u32 alt_result[8]; - - alt_result[0] = tmps[gid].alt_result[0]; - alt_result[1] = tmps[gid].alt_result[1]; - alt_result[2] = tmps[gid].alt_result[2]; - alt_result[3] = tmps[gid].alt_result[3]; - alt_result[4] = tmps[gid].alt_result[4]; - alt_result[5] = tmps[gid].alt_result[5]; - alt_result[6] = tmps[gid].alt_result[6]; - alt_result[7] = tmps[gid].alt_result[7]; - - u32 salt_len = salt_bufs[salt_pos].salt_len; - - /* Repeatedly run the collected hash value through SHA256 to burn - CPU cycles. */ - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - u32 tmp[8]; - - init_ctx (tmp); - - u32 block[32]; - - bzero16 (&block[ 0]); - bzero16 (&block[16]); - - u32 block_len = 0; - - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 1 : 0; - const u32 j7 = (j % 7) ? 1 : 0; - - if (j1) - { - block[0] = p_bytes[0]; - block[1] = p_bytes[1]; - block[2] = p_bytes[2]; - block[3] = p_bytes[3]; - - block_len = pw_len; - } - else - { - block[0] = alt_result[0]; - block[1] = alt_result[1]; - block[2] = alt_result[2]; - block[3] = alt_result[3]; - block[4] = alt_result[4]; - block[5] = alt_result[5]; - block[6] = alt_result[6]; - block[7] = alt_result[7]; - - block_len = 32; - } - - if (j3) - { - block_len = memcat20 (block, block_len, s_bytes, salt_len); - } - - if (j7) - { - block_len = memcat20 (block, block_len, p_bytes, pw_len); - } - - if (j1) - { - block_len = memcat20 (block, block_len, &alt_result[0], 16); - block_len = memcat20_x80 (block, block_len, &alt_result[4], 16); - } - else - { - block_len = memcat20 (block, block_len, p_bytes_x80, pw_len); - } - - if (block_len >= 56) - { - sha256_transform (block, tmp); - - block[ 0] = block[16]; - block[ 1] = block[17]; - block[ 2] = block[18]; - block[ 3] = block[19]; - block[ 4] = 0; - block[ 5] = 0; - block[ 6] = 0; - block[ 7] = 0; - block[ 8] = 0; - block[ 9] = 0; - block[10] = 0; - block[11] = 0; - block[12] = 0; - block[13] = 0; - block[14] = 0; - block[15] = 0; - } - - block[15] = swap32 (block_len * 8); - - sha256_transform_no14 (block, tmp); - - bswap8 (tmp); - - alt_result[0] = tmp[0]; - alt_result[1] = tmp[1]; - alt_result[2] = tmp[2]; - alt_result[3] = tmp[3]; - alt_result[4] = tmp[4]; - alt_result[5] = tmp[5]; - alt_result[6] = tmp[6]; - alt_result[7] = tmp[7]; - } - - tmps[gid].alt_result[0] = alt_result[0]; - tmps[gid].alt_result[1] = alt_result[1]; - tmps[gid].alt_result[2] = alt_result[2]; - tmps[gid].alt_result[3] = alt_result[3]; - tmps[gid].alt_result[4] = alt_result[4]; - tmps[gid].alt_result[5] = alt_result[5]; - tmps[gid].alt_result[6] = alt_result[6]; - tmps[gid].alt_result[7] = alt_result[7]; -} - -__kernel void m07400_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - const u32 r0 = tmps[gid].alt_result[0]; - const u32 r1 = tmps[gid].alt_result[1]; - const u32 r2 = tmps[gid].alt_result[2]; - const u32 r3 = tmps[gid].alt_result[3]; - - #define il_pos 0 - - #include COMPARE_M -} - -#else - -// this is basically a much cleaner version, but apparently drops speeds by over 100% :( - -#define PUTCHAR32_BE(a,p,c) ((u8 *)(a))[(p) ^ 3] = (u8) (c) -#define GETCHAR32_BE(a,p) ((u8 *)(a))[(p) ^ 3] - -typedef struct -{ - u32 state[8]; - u32 buf[32]; - int len; - -} sha256_ctx_t; - -void sha256_transform (const u32 w[16], u32 digest[8]) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - u32 e = digest[4]; - u32 f = digest[5]; - u32 g = digest[6]; - u32 h = digest[7]; - - u32 w0_t = w[ 0]; - u32 w1_t = w[ 1]; - u32 w2_t = w[ 2]; - u32 w3_t = w[ 3]; - u32 w4_t = w[ 4]; - u32 w5_t = w[ 5]; - u32 w6_t = w[ 6]; - u32 w7_t = w[ 7]; - u32 w8_t = w[ 8]; - u32 w9_t = w[ 9]; - u32 wa_t = w[10]; - u32 wb_t = w[11]; - u32 wc_t = w[12]; - u32 wd_t = w[13]; - u32 we_t = w[14]; - u32 wf_t = w[15]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void sha256_init (sha256_ctx_t *sha256_ctx) -{ - sha256_ctx->state[0] = SHA256M_A; - sha256_ctx->state[1] = SHA256M_B; - sha256_ctx->state[2] = SHA256M_C; - sha256_ctx->state[3] = SHA256M_D; - sha256_ctx->state[4] = SHA256M_E; - sha256_ctx->state[5] = SHA256M_F; - sha256_ctx->state[6] = SHA256M_G; - sha256_ctx->state[7] = SHA256M_H; - - sha256_ctx->len = 0; -} - -void sha256_update (sha256_ctx_t *sha256_ctx, const u32 *buf, int len) -{ - int pos = sha256_ctx->len & 0x3f; - - sha256_ctx->len += len; - - if ((pos + len) < 64) - { - for (int i = 0; i < len; i++) - { - PUTCHAR32_BE (sha256_ctx->buf, pos++, GETCHAR32_BE (buf, i)); - } - - return; - } - - int cnt = 64 - pos; - - for (int i = 0; i < cnt; i++) - { - PUTCHAR32_BE (sha256_ctx->buf, pos++, GETCHAR32_BE (buf, i)); - } - - sha256_transform (sha256_ctx->buf, sha256_ctx->state); - - len -= cnt; - - for (int i = 0; i < len; i++) - { - PUTCHAR32_BE (sha256_ctx->buf, i, GETCHAR32_BE (buf, cnt + i)); - } -} - -void sha256_final (sha256_ctx_t *sha256_ctx) -{ - int pos = sha256_ctx->len & 0x3f; - - for (int i = pos; i < 64; i++) - { - PUTCHAR32_BE (sha256_ctx->buf, i, 0); - } - - PUTCHAR32_BE (sha256_ctx->buf, pos, 0x80); - - if (pos >= 56) - { - sha256_transform (sha256_ctx->buf, sha256_ctx->state); - - sha256_ctx->buf[ 0] = 0; - sha256_ctx->buf[ 1] = 0; - sha256_ctx->buf[ 2] = 0; - sha256_ctx->buf[ 3] = 0; - sha256_ctx->buf[ 4] = 0; - sha256_ctx->buf[ 5] = 0; - sha256_ctx->buf[ 6] = 0; - sha256_ctx->buf[ 7] = 0; - sha256_ctx->buf[ 8] = 0; - sha256_ctx->buf[ 9] = 0; - sha256_ctx->buf[10] = 0; - sha256_ctx->buf[11] = 0; - sha256_ctx->buf[12] = 0; - sha256_ctx->buf[13] = 0; - sha256_ctx->buf[14] = 0; - sha256_ctx->buf[15] = 0; - } - - sha256_ctx->buf[15] = sha256_ctx->len * 8; - - sha256_transform (sha256_ctx->buf, sha256_ctx->state); -} - -__kernel void m07400_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw[4]; - - pw[0] = swap32 (pws[gid].i[0]); - pw[1] = swap32 (pws[gid].i[1]); - pw[2] = swap32 (pws[gid].i[2]); - pw[3] = swap32 (pws[gid].i[3]); - - const u32 pw_len = pws[gid].pw_len; - - /** - * salt - */ - - u32 salt[4]; - - salt[0] = swap32 (salt_bufs[salt_pos].salt_buf[0]); - salt[1] = swap32 (salt_bufs[salt_pos].salt_buf[1]); - salt[2] = swap32 (salt_bufs[salt_pos].salt_buf[2]); - salt[3] = swap32 (salt_bufs[salt_pos].salt_buf[3]); - - u32 salt_len = salt_bufs[salt_pos].salt_len; - - /** - * begin - */ - - sha256_ctx_t sha256_ctx; - - sha256_init (&sha256_ctx); - - sha256_update (&sha256_ctx, pw, pw_len); - sha256_update (&sha256_ctx, salt, salt_len); - sha256_update (&sha256_ctx, pw, pw_len); - - sha256_final (&sha256_ctx); - - u32 tmp[8]; - - tmp[0] = sha256_ctx.state[0]; - tmp[1] = sha256_ctx.state[1]; - tmp[2] = sha256_ctx.state[2]; - tmp[3] = sha256_ctx.state[3]; - tmp[4] = sha256_ctx.state[4]; - tmp[5] = sha256_ctx.state[5]; - tmp[6] = sha256_ctx.state[6]; - tmp[7] = sha256_ctx.state[7]; - - sha256_init (&sha256_ctx); - - sha256_update (&sha256_ctx, pw, pw_len); - sha256_update (&sha256_ctx, salt, salt_len); - sha256_update (&sha256_ctx, tmp, pw_len); - - for (u32 j = pw_len; j; j >>= 1) - { - if (j & 1) - { - sha256_update (&sha256_ctx, tmp, 32); - } - else - { - sha256_update (&sha256_ctx, pw, pw_len); - } - } - - sha256_final (&sha256_ctx); - - tmps[gid].alt_result[0] = sha256_ctx.state[0]; - tmps[gid].alt_result[1] = sha256_ctx.state[1]; - tmps[gid].alt_result[2] = sha256_ctx.state[2]; - tmps[gid].alt_result[3] = sha256_ctx.state[3]; - tmps[gid].alt_result[4] = sha256_ctx.state[4]; - tmps[gid].alt_result[5] = sha256_ctx.state[5]; - tmps[gid].alt_result[6] = sha256_ctx.state[6]; - tmps[gid].alt_result[7] = sha256_ctx.state[7]; - - // p_bytes - - sha256_init (&sha256_ctx); - - for (u32 j = 0; j < pw_len; j++) - { - sha256_update (&sha256_ctx, pw, pw_len); - } - - sha256_final (&sha256_ctx); - - tmps[gid].p_bytes[0] = sha256_ctx.state[0]; - tmps[gid].p_bytes[1] = sha256_ctx.state[1]; - tmps[gid].p_bytes[2] = sha256_ctx.state[2]; - tmps[gid].p_bytes[3] = sha256_ctx.state[3]; - - // s_bytes - - sha256_init (&sha256_ctx); - - for (u32 j = 0; j < 16 + ((tmps[gid].alt_result[0] >> 24) & 0xff); j++) - { - sha256_update (&sha256_ctx, salt, salt_len); - } - - sha256_final (&sha256_ctx); - - tmps[gid].s_bytes[0] = sha256_ctx.state[0]; - tmps[gid].s_bytes[1] = sha256_ctx.state[1]; - tmps[gid].s_bytes[2] = sha256_ctx.state[2]; - tmps[gid].s_bytes[3] = sha256_ctx.state[3]; -} - -__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 p_bytes0[4]; - - p_bytes0[0] = tmps[gid].p_bytes[0]; - p_bytes0[1] = tmps[gid].p_bytes[1]; - p_bytes0[2] = tmps[gid].p_bytes[2]; - p_bytes0[3] = tmps[gid].p_bytes[3]; - - const u32 pw_len = pws[gid].pw_len; - - u32 s_bytes0[4]; - - s_bytes0[0] = tmps[gid].s_bytes[0]; - s_bytes0[1] = tmps[gid].s_bytes[1]; - s_bytes0[2] = tmps[gid].s_bytes[2]; - s_bytes0[3] = tmps[gid].s_bytes[3]; - const u32 salt_len = salt_bufs[salt_pos].salt_len; - u32 wpc_len[8]; + const u32 salt_lenv = ceil ((float) salt_len / 4); - wpc_len[0] = 32 + 0 + 0 + pw_len; - wpc_len[1] = pw_len + 0 + 0 + 32; - wpc_len[2] = 32 + salt_len + 0 + pw_len; - wpc_len[3] = pw_len + salt_len + 0 + 32; - wpc_len[4] = 32 + 0 + pw_len + pw_len; - wpc_len[5] = pw_len + 0 + pw_len + 32; - wpc_len[6] = 32 + salt_len + pw_len + pw_len; - wpc_len[7] = pw_len + salt_len + pw_len + 32; + u32 s[64] = { 0 }; - u32 wpc[8][32] = { { 0 } }; - - for (u32 i = 0; i < 8; i++) + for (int idx = 0; idx < salt_lenv; idx++) { - u32 block_len = 0; + s[idx] = salt_bufs[salt_pos].salt_buf[idx]; - if (i & 1) + barrier (CLK_GLOBAL_MEM_FENCE); + } + + for (int idx = 0; idx < salt_lenv; idx++) + { + s[idx] = swap32_S (s[idx]); + } + + /** + * prepare + */ + + sha256_ctx_t ctx; + + sha256_init (&ctx); + + sha256_update (&ctx, w, pw_len); + + sha256_update (&ctx, s, salt_len); + + sha256_update (&ctx, w, pw_len); + + sha256_final (&ctx); + + u32 final[16] = { 0 }; + + final[0] = ctx.h[0]; + final[1] = ctx.h[1]; + final[2] = ctx.h[2]; + final[3] = ctx.h[3]; + final[4] = ctx.h[4]; + final[5] = ctx.h[5]; + final[6] = ctx.h[6]; + final[7] = ctx.h[7]; + + // alt_result + + sha256_init (&ctx); + + sha256_update (&ctx, w, pw_len); + + sha256_update (&ctx, s, salt_len); + + int pl; + + for (pl = pw_len; pl > 32; pl -= 32) + { + sha256_update (&ctx, final, 32); + } + + u32 t_final[16] = { 0 }; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 8; i++) t_final[i] = final[i]; + + truncate_block_16x4_be (t_final + 0, t_final + 4, t_final + 8, t_final + 12, pl); + + sha256_update (&ctx, t_final, pl); + + for (int cnt = pw_len; cnt > 0; cnt >>= 1) + { + if ((cnt & 1) != 0) { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); - } + sha256_update (&ctx, final, 32); } else { - block_len += 32; - } - - if (i & 2) - { - for (u32 j = 0; j < salt_len; j++) - { - PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (s_bytes0, j)); - } - } - - if (i & 4) - { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); - } - } - - if (i & 1) - { - block_len += 32; - } - else - { - for (u32 j = 0; j < pw_len; j++) - { - PUTCHAR32_BE (wpc[i], block_len++, GETCHAR32_BE (p_bytes0, j)); - } - } - - PUTCHAR32_BE (wpc[i], block_len, 0x80); - - if (block_len < 56) - { - wpc[i][15] = block_len * 8; - } - else - { - wpc[i][31] = block_len * 8; + sha256_update (&ctx, w, pw_len); } } + sha256_final (&ctx); + + tmps[gid].alt_result[0] = ctx.h[0]; + tmps[gid].alt_result[1] = ctx.h[1]; + tmps[gid].alt_result[2] = ctx.h[2]; + tmps[gid].alt_result[3] = ctx.h[3]; + tmps[gid].alt_result[4] = ctx.h[4]; + tmps[gid].alt_result[5] = ctx.h[5]; + tmps[gid].alt_result[6] = ctx.h[6]; + tmps[gid].alt_result[7] = ctx.h[7]; + + // p_bytes + + sha256_init (&ctx); + + for (u32 j = 0; j < pw_len; j++) + { + sha256_update (&ctx, w, pw_len); + } + + sha256_final (&ctx); + + final[ 0] = ctx.h[0]; + final[ 1] = ctx.h[1]; + final[ 2] = ctx.h[2]; + final[ 3] = ctx.h[3]; + final[ 4] = ctx.h[4]; + final[ 5] = ctx.h[5]; + final[ 6] = ctx.h[6]; + final[ 7] = ctx.h[7]; + final[ 8] = 0; + final[ 9] = 0; + final[10] = 0; + final[11] = 0; + final[12] = 0; + final[13] = 0; + final[14] = 0; + final[15] = 0; + + u32 p_final[64] = { 0 }; + + int idx; + + for (pl = pw_len, idx = 0; pl > 32; pl -= 32, idx += 8) + { + p_final[idx + 0] = final[0]; + p_final[idx + 1] = final[1]; + p_final[idx + 2] = final[2]; + p_final[idx + 3] = final[3]; + p_final[idx + 4] = final[4]; + p_final[idx + 5] = final[5]; + p_final[idx + 6] = final[6]; + p_final[idx + 7] = final[7]; + } + + truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); + + p_final[idx + 0] = final[0]; + p_final[idx + 1] = final[1]; + p_final[idx + 2] = final[2]; + p_final[idx + 3] = final[3]; + p_final[idx + 4] = final[4]; + p_final[idx + 5] = final[5]; + p_final[idx + 6] = final[6]; + p_final[idx + 7] = final[7]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) tmps[gid].p_bytes[i] = p_final[i]; + + // s_bytes + + sha256_init (&ctx); + + for (u32 j = 0; j < 16 + (tmps[gid].alt_result[0] >> 24); j++) + { + sha256_update (&ctx, s, salt_len); + } + + sha256_final (&ctx); + + final[ 0] = ctx.h[0]; + final[ 1] = ctx.h[1]; + final[ 2] = ctx.h[2]; + final[ 3] = ctx.h[3]; + final[ 4] = ctx.h[4]; + final[ 5] = ctx.h[5]; + final[ 6] = ctx.h[6]; + final[ 7] = ctx.h[7]; + final[ 8] = 0; + final[ 9] = 0; + final[10] = 0; + final[11] = 0; + final[12] = 0; + final[13] = 0; + final[14] = 0; + final[15] = 0; + + u32 s_final[64] = { 0 }; + + for (pl = salt_len, idx = 0; pl > 32; pl -= 32, idx += 8) + { + s_final[idx + 0] = final[0]; + s_final[idx + 1] = final[1]; + s_final[idx + 2] = final[2]; + s_final[idx + 3] = final[3]; + s_final[idx + 4] = final[4]; + s_final[idx + 5] = final[5]; + s_final[idx + 6] = final[6]; + s_final[idx + 7] = final[7]; + } + + truncate_block_16x4_be (final + 0, final + 4, final + 8, final + 12, pl); + + s_final[idx + 0] = final[0]; + s_final[idx + 1] = final[1]; + s_final[idx + 2] = final[2]; + s_final[idx + 3] = final[3]; + s_final[idx + 4] = final[4]; + s_final[idx + 5] = final[5]; + s_final[idx + 6] = final[6]; + s_final[idx + 7] = final[7]; + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 0; i < 64; i++) tmps[gid].s_bytes[i] = s_final[i]; +} + +__kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global sha256crypt_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ /** * base */ - u32 alt_result[8]; + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 pw_len = pws[gid].pw_len; + + const u32 salt_len = salt_bufs[salt_pos].salt_len; + + u32 alt_result[16] = { 0 }; alt_result[0] = tmps[gid].alt_result[0]; alt_result[1] = tmps[gid].alt_result[1]; @@ -1633,91 +283,53 @@ __kernel void m07400_loop (__global pw_t *pws, __global const kernel_rule_t *rul alt_result[6] = tmps[gid].alt_result[6]; alt_result[7] = tmps[gid].alt_result[7]; - /* Repeatedly run the collected hash value through SHA256 to burn + /* Repeatedly run the collected hash value through sha256 to burn CPU cycles. */ for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - const u32 j1 = (j & 1) ? 1 : 0; - const u32 j3 = (j % 3) ? 2 : 0; - const u32 j7 = (j % 7) ? 4 : 0; + sha256_ctx_t ctx; - const u32 pc = j1 + j3 + j7; + sha256_init (&ctx); - u32 block[32]; - - block[ 0] = wpc[pc][ 0]; - block[ 1] = wpc[pc][ 1]; - block[ 2] = wpc[pc][ 2]; - block[ 3] = wpc[pc][ 3]; - block[ 4] = wpc[pc][ 4]; - block[ 5] = wpc[pc][ 5]; - block[ 6] = wpc[pc][ 6]; - block[ 7] = wpc[pc][ 7]; - block[ 8] = wpc[pc][ 8]; - block[ 9] = wpc[pc][ 9]; - block[10] = wpc[pc][10]; - block[11] = wpc[pc][11]; - block[12] = wpc[pc][12]; - block[13] = wpc[pc][13]; - block[14] = wpc[pc][14]; - block[15] = wpc[pc][15]; - block[16] = wpc[pc][16]; - block[17] = wpc[pc][17]; - block[18] = wpc[pc][18]; - block[19] = wpc[pc][19]; - block[20] = wpc[pc][20]; - block[21] = wpc[pc][21]; - block[22] = wpc[pc][22]; - block[23] = wpc[pc][23]; - block[24] = wpc[pc][24]; - block[25] = wpc[pc][25]; - block[26] = wpc[pc][26]; - block[27] = wpc[pc][27]; - block[28] = wpc[pc][28]; - block[29] = wpc[pc][29]; - block[30] = wpc[pc][30]; - block[31] = wpc[pc][31]; - - const u32 block_len = wpc_len[pc]; - - if (j1) + if (j & 1) { - #ifdef _unroll - #pragma unroll - #endif - for (u32 k = 0, p = block_len - 32; k < 32; k++, p++) - { - PUTCHAR32_BE (block, p, GETCHAR32_BE (alt_result, k)); - } + sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); } - else + else { - block[0] = alt_result[0]; - block[1] = alt_result[1]; - block[2] = alt_result[2]; - block[3] = alt_result[3]; - block[4] = alt_result[4]; - block[5] = alt_result[5]; - block[6] = alt_result[6]; - block[7] = alt_result[7]; + sha256_update (&ctx, alt_result, 32); } - alt_result[0] = SHA256M_A; - alt_result[1] = SHA256M_B; - alt_result[2] = SHA256M_C; - alt_result[3] = SHA256M_D; - alt_result[4] = SHA256M_E; - alt_result[5] = SHA256M_F; - alt_result[6] = SHA256M_G; - alt_result[7] = SHA256M_H; - - sha256_transform (block, alt_result); - - if (block_len >= 56) + if (j % 3) { - sha256_transform (block + 16, alt_result); + sha256_update_global (&ctx, tmps[gid].s_bytes, salt_len); } + + if (j % 7) + { + sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); + } + + if (j & 1) + { + sha256_update (&ctx, alt_result, 32); + } + else + { + sha256_update_global (&ctx, tmps[gid].p_bytes, pw_len); + } + + sha256_final (&ctx); + + alt_result[0] = ctx.h[0]; + alt_result[1] = ctx.h[1]; + alt_result[2] = ctx.h[2]; + alt_result[3] = ctx.h[3]; + alt_result[4] = ctx.h[4]; + alt_result[5] = ctx.h[5]; + alt_result[6] = ctx.h[6]; + alt_result[7] = ctx.h[7]; } tmps[gid].alt_result[0] = alt_result[0]; @@ -1742,14 +354,12 @@ __kernel void m07400_comp (__global pw_t *pws, __global const kernel_rule_t *rul const u32 lid = get_local_id (0); - const u32 r0 = swap32 (tmps[gid].alt_result[0]); - const u32 r1 = swap32 (tmps[gid].alt_result[1]); - const u32 r2 = swap32 (tmps[gid].alt_result[2]); - const u32 r3 = swap32 (tmps[gid].alt_result[3]); + const u32 r0 = swap32_S (tmps[gid].alt_result[0]); + const u32 r1 = swap32_S (tmps[gid].alt_result[1]); + const u32 r2 = swap32_S (tmps[gid].alt_result[2]); + const u32 r3 = swap32_S (tmps[gid].alt_result[3]); #define il_pos 0 #include COMPARE_M } - -#endif diff --git a/OpenCL/m10700-optimized.cl b/OpenCL/m10700-optimized.cl new file mode 100644 index 000000000..297b33917 --- /dev/null +++ b/OpenCL/m10700-optimized.cl @@ -0,0 +1,826 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_hash_sha256.cl" +#include "inc_hash_sha384.cl" +#include "inc_hash_sha512.cl" +#include "inc_cipher_aes.cl" + +#define COMPARE_S "inc_comp_single.cl" +#define COMPARE_M "inc_comp_multi.cl" + +typedef struct +{ + union + { + u32 dgst32[16]; + u64 dgst64[8]; + }; + + u32 dgst_len; + + union + { + u32 W32[32]; + u64 W64[16]; + }; + + u32 W_len; + +} ctx_t; + +void orig_sha256_transform (const u32 *w0, const u32 *w1, const u32 *w2, const u32 *w3, u32 *digest) +{ + u32 a = digest[0]; + u32 b = digest[1]; + u32 c = digest[2]; + u32 d = digest[3]; + u32 e = digest[4]; + u32 f = digest[5]; + u32 g = digest[6]; + u32 h = digest[7]; + + u32 w0_t = swap32 (w0[0]); + u32 w1_t = swap32 (w0[1]); + u32 w2_t = swap32 (w0[2]); + u32 w3_t = swap32 (w0[3]); + u32 w4_t = swap32 (w1[0]); + u32 w5_t = swap32 (w1[1]); + u32 w6_t = swap32 (w1[2]); + u32 w7_t = swap32 (w1[3]); + u32 w8_t = swap32 (w2[0]); + u32 w9_t = swap32 (w2[1]); + u32 wa_t = swap32 (w2[2]); + u32 wb_t = swap32 (w2[3]); + u32 wc_t = swap32 (w3[0]); + u32 wd_t = swap32 (w3[1]); + u32 we_t = swap32 (w3[2]); + u32 wf_t = swap32 (w3[3]); + + #define ROUND256_EXPAND() \ + { \ + w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND256_STEP(i) \ + { \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ + SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ + } + + ROUND256_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 64; i += 16) + { + ROUND256_EXPAND (); ROUND256_STEP (i); + } + + #undef ROUND256_EXPAND + #undef ROUND256_STEP + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void orig_sha384_transform (const u64 *w0, const u64 *w1, const u64 *w2, const u64 *w3, u64 *digest) +{ + u64 a = digest[0]; + u64 b = digest[1]; + u64 c = digest[2]; + u64 d = digest[3]; + u64 e = digest[4]; + u64 f = digest[5]; + u64 g = digest[6]; + u64 h = digest[7]; + + u64 w0_t = swap64 (w0[0]); + u64 w1_t = swap64 (w0[1]); + u64 w2_t = swap64 (w0[2]); + u64 w3_t = swap64 (w0[3]); + u64 w4_t = swap64 (w1[0]); + u64 w5_t = swap64 (w1[1]); + u64 w6_t = swap64 (w1[2]); + u64 w7_t = swap64 (w1[3]); + u64 w8_t = swap64 (w2[0]); + u64 w9_t = swap64 (w2[1]); + u64 wa_t = swap64 (w2[2]); + u64 wb_t = swap64 (w2[3]); + u64 wc_t = swap64 (w3[0]); + u64 wd_t = swap64 (w3[1]); + u64 we_t = swap64 (w3[2]); + u64 wf_t = swap64 (w3[3]); + + #define ROUND384_EXPAND() \ + { \ + w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND384_STEP(i) \ + { \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ + } + + ROUND384_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND384_EXPAND (); ROUND384_STEP (i); + } + + #undef ROUND384_EXPAND + #undef ROUND384_STEP + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +void orig_sha512_transform (const u64 *w0, const u64 *w1, const u64 *w2, const u64 *w3, u64 *digest) +{ + u64 a = digest[0]; + u64 b = digest[1]; + u64 c = digest[2]; + u64 d = digest[3]; + u64 e = digest[4]; + u64 f = digest[5]; + u64 g = digest[6]; + u64 h = digest[7]; + + u64 w0_t = swap64 (w0[0]); + u64 w1_t = swap64 (w0[1]); + u64 w2_t = swap64 (w0[2]); + u64 w3_t = swap64 (w0[3]); + u64 w4_t = swap64 (w1[0]); + u64 w5_t = swap64 (w1[1]); + u64 w6_t = swap64 (w1[2]); + u64 w7_t = swap64 (w1[3]); + u64 w8_t = swap64 (w2[0]); + u64 w9_t = swap64 (w2[1]); + u64 wa_t = swap64 (w2[2]); + u64 wb_t = swap64 (w2[3]); + u64 wc_t = swap64 (w3[0]); + u64 wd_t = swap64 (w3[1]); + u64 we_t = swap64 (w3[2]); + u64 wf_t = swap64 (w3[3]); + + #define ROUND512_EXPAND() \ + { \ + w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND512_STEP(i) \ + { \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ + SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ + } + + ROUND512_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND512_EXPAND (); ROUND512_STEP (i); + } + + #undef ROUND512_EXPAND + #undef ROUND512_STEP + + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; +} + +#define AESSZ 16 // AES_BLOCK_SIZE + +#define BLSZ256 32 +#define BLSZ384 48 +#define BLSZ512 64 + +#define WORDSZ256 64 +#define WORDSZ384 128 +#define WORDSZ512 128 + +#define PWMAXSZ 32 // hashcat password length limit +#define BLMAXSZ BLSZ512 +#define WORDMAXSZ WORDSZ512 + +#define PWMAXSZ4 (PWMAXSZ / 4) +#define BLMAXSZ4 (BLMAXSZ / 4) +#define WORDMAXSZ4 (WORDMAXSZ / 4) +#define AESSZ4 (AESSZ / 4) + +void make_sc (u32 *sc, const u32 *pw, const u32 pw_len, const u32 *bl, const u32 bl_len) +{ + const u32 bd = bl_len / 4; + + const u32 pm = pw_len % 4; + const u32 pd = pw_len / 4; + + u32 idx = 0; + + if (pm == 0) + { + for (u32 i = 0; i < pd; i++) sc[idx++] = pw[i]; + for (u32 i = 0; i < bd; i++) sc[idx++] = bl[i]; + for (u32 i = 0; i < 4; i++) sc[idx++] = sc[i]; + } + else + { + u32 pm4 = 4 - pm; + + u32 i; + + #if defined IS_AMD || defined IS_GENERIC + for (i = 0; i < pd; i++) sc[idx++] = pw[i]; + sc[idx++] = pw[i] + | amd_bytealign (bl[0], 0, pm4); + for (i = 1; i < bd; i++) sc[idx++] = amd_bytealign (bl[i], bl[i - 1], pm4); + sc[idx++] = amd_bytealign (sc[0], bl[i - 1], pm4); + for (i = 1; i < 4; i++) sc[idx++] = amd_bytealign (sc[i], sc[i - 1], pm4); + sc[idx++] = amd_bytealign ( 0, sc[i - 1], pm4); + #endif + + #ifdef IS_NV + int selector = (0x76543210 >> (pm4 * 4)) & 0xffff; + + for (i = 0; i < pd; i++) sc[idx++] = pw[i]; + sc[idx++] = pw[i] + | __byte_perm ( 0, bl[0], selector); + for (i = 1; i < bd; i++) sc[idx++] = __byte_perm (bl[i - 1], bl[i], selector); + sc[idx++] = __byte_perm (bl[i - 1], sc[0], selector); + for (i = 1; i < 4; i++) sc[idx++] = __byte_perm (sc[i - 1], sc[i], selector); + sc[idx++] = __byte_perm (sc[i - 1], 0, selector); + #endif + } +} + +void make_pt_with_offset (u32 *pt, const u32 offset, const u32 *sc, const u32 pwbl_len) +{ + const u32 m = offset % pwbl_len; + + const u32 om = m % 4; + const u32 od = m / 4; + + #if defined IS_AMD || defined IS_GENERIC + pt[0] = amd_bytealign (sc[od + 1], sc[od + 0], om); + pt[1] = amd_bytealign (sc[od + 2], sc[od + 1], om); + pt[2] = amd_bytealign (sc[od + 3], sc[od + 2], om); + pt[3] = amd_bytealign (sc[od + 4], sc[od + 3], om); + #endif + + #ifdef IS_NV + int selector = (0x76543210 >> (om * 4)) & 0xffff; + + pt[0] = __byte_perm (sc[od + 0], sc[od + 1], selector); + pt[1] = __byte_perm (sc[od + 1], sc[od + 2], selector); + pt[2] = __byte_perm (sc[od + 2], sc[od + 3], selector); + pt[3] = __byte_perm (sc[od + 3], sc[od + 4], selector); + #endif +} + +void make_w_with_offset (ctx_t *ctx, const u32 W_len, const u32 offset, const u32 *sc, const u32 pwbl_len, u32 *iv, const u32 *ks, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + for (u32 k = 0, wk = 0; k < W_len; k += AESSZ, wk += AESSZ4) + { + u32 pt[AESSZ4]; + + make_pt_with_offset (pt, offset + k, sc, pwbl_len); + + pt[0] ^= iv[0]; + pt[1] ^= iv[1]; + pt[2] ^= iv[2]; + pt[3] ^= iv[3]; + + aes128_encrypt (ks, pt, iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + ctx->W32[wk + 0] = iv[0]; + ctx->W32[wk + 1] = iv[1]; + ctx->W32[wk + 2] = iv[2]; + ctx->W32[wk + 3] = iv[3]; + } +} + +u32 do_round (const u32 *pw, const u32 pw_len, ctx_t *ctx, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + // make scratch buffer + + u32 sc[PWMAXSZ4 + BLMAXSZ4 + AESSZ4]; + + make_sc (sc, pw, pw_len, ctx->dgst32, ctx->dgst_len); + + // make sure pwbl_len is calculcated before it gets changed + + const u32 pwbl_len = pw_len + ctx->dgst_len; + + // init iv + + u32 iv[AESSZ4]; + + iv[0] = ctx->dgst32[4]; + iv[1] = ctx->dgst32[5]; + iv[2] = ctx->dgst32[6]; + iv[3] = ctx->dgst32[7]; + + // init aes + + u32 ks[44]; + + aes128_set_encrypt_key (ks, ctx->dgst32, s_te0, s_te1, s_te2, s_te3, s_te4); + + // first call is special as the hash depends on the result of it + // but since we do not know about the outcome at this time + // we must use the max + + make_w_with_offset (ctx, WORDMAXSZ, 0, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + + // now we can find out hash to use + + u32 sum = 0; + + for (u32 i = 0; i < 4; i++) + { + sum += (ctx->W32[i] >> 24) & 0xff; + sum += (ctx->W32[i] >> 16) & 0xff; + sum += (ctx->W32[i] >> 8) & 0xff; + sum += (ctx->W32[i] >> 0) & 0xff; + } + + // init hash + + switch (sum % 3) + { + case 0: ctx->dgst32[0] = SHA256M_A; + ctx->dgst32[1] = SHA256M_B; + ctx->dgst32[2] = SHA256M_C; + ctx->dgst32[3] = SHA256M_D; + ctx->dgst32[4] = SHA256M_E; + ctx->dgst32[5] = SHA256M_F; + ctx->dgst32[6] = SHA256M_G; + ctx->dgst32[7] = SHA256M_H; + ctx->dgst_len = BLSZ256; + ctx->W_len = WORDSZ256; + orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); + orig_sha256_transform (&ctx->W32[16], &ctx->W32[20], &ctx->W32[24], &ctx->W32[28], ctx->dgst32); + break; + case 1: ctx->dgst64[0] = SHA384M_A; + ctx->dgst64[1] = SHA384M_B; + ctx->dgst64[2] = SHA384M_C; + ctx->dgst64[3] = SHA384M_D; + ctx->dgst64[4] = SHA384M_E; + ctx->dgst64[5] = SHA384M_F; + ctx->dgst64[6] = SHA384M_G; + ctx->dgst64[7] = SHA384M_H; + ctx->dgst_len = BLSZ384; + ctx->W_len = WORDSZ384; + orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + break; + case 2: ctx->dgst64[0] = SHA512M_A; + ctx->dgst64[1] = SHA512M_B; + ctx->dgst64[2] = SHA512M_C; + ctx->dgst64[3] = SHA512M_D; + ctx->dgst64[4] = SHA512M_E; + ctx->dgst64[5] = SHA512M_F; + ctx->dgst64[6] = SHA512M_G; + ctx->dgst64[7] = SHA512M_H; + ctx->dgst_len = BLSZ512; + ctx->W_len = WORDSZ512; + orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + break; + } + + // main loop + + const u32 final_len = pwbl_len * 64; + + const u32 iter_max = ctx->W_len - (ctx->W_len / 8); + + u32 offset; + u32 left; + + for (offset = WORDMAXSZ, left = final_len - offset; left >= iter_max; offset += ctx->W_len, left -= ctx->W_len) + { + make_w_with_offset (ctx, ctx->W_len, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + + switch (ctx->dgst_len) + { + case BLSZ256: orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); + break; + case BLSZ384: orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + break; + case BLSZ512: orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + break; + } + } + + u32 ex = 0; + + if (left) + { + switch (ctx->dgst_len) + { + case BLSZ384: make_w_with_offset (ctx, 64, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + ctx->W64[ 8] = 0x80; + ctx->W64[ 9] = 0; + ctx->W64[10] = 0; + ctx->W64[11] = 0; + ctx->W64[12] = 0; + ctx->W64[13] = 0; + ctx->W64[14] = 0; + ctx->W64[15] = swap64 ((u64) (final_len * 8)); + ex = ctx->W64[7] >> 56; + break; + case BLSZ512: make_w_with_offset (ctx, 64, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + ctx->W64[ 8] = 0x80; + ctx->W64[ 9] = 0; + ctx->W64[10] = 0; + ctx->W64[11] = 0; + ctx->W64[12] = 0; + ctx->W64[13] = 0; + ctx->W64[14] = 0; + ctx->W64[15] = swap64 ((u64) (final_len * 8)); + ex = ctx->W64[7] >> 56; + break; + } + } + else + { + switch (ctx->dgst_len) + { + case BLSZ256: ex = ctx->W32[15] >> 24; + ctx->W32[ 0] = 0x80; + ctx->W32[ 1] = 0; + ctx->W32[ 2] = 0; + ctx->W32[ 3] = 0; + ctx->W32[ 4] = 0; + ctx->W32[ 5] = 0; + ctx->W32[ 6] = 0; + ctx->W32[ 7] = 0; + ctx->W32[ 8] = 0; + ctx->W32[ 9] = 0; + ctx->W32[10] = 0; + ctx->W32[11] = 0; + ctx->W32[12] = 0; + ctx->W32[13] = 0; + ctx->W32[14] = 0; + ctx->W32[15] = swap32 (final_len * 8); + break; + case BLSZ384: ex = ctx->W64[15] >> 56; + ctx->W64[ 0] = 0x80; + ctx->W64[ 1] = 0; + ctx->W64[ 2] = 0; + ctx->W64[ 3] = 0; + ctx->W64[ 4] = 0; + ctx->W64[ 5] = 0; + ctx->W64[ 6] = 0; + ctx->W64[ 7] = 0; + ctx->W64[ 8] = 0; + ctx->W64[ 9] = 0; + ctx->W64[10] = 0; + ctx->W64[11] = 0; + ctx->W64[12] = 0; + ctx->W64[13] = 0; + ctx->W64[14] = 0; + ctx->W64[15] = swap64 ((u64) (final_len * 8)); + break; + case BLSZ512: ex = ctx->W64[15] >> 56; + ctx->W64[ 0] = 0x80; + ctx->W64[ 1] = 0; + ctx->W64[ 2] = 0; + ctx->W64[ 3] = 0; + ctx->W64[ 4] = 0; + ctx->W64[ 5] = 0; + ctx->W64[ 6] = 0; + ctx->W64[ 7] = 0; + ctx->W64[ 8] = 0; + ctx->W64[ 9] = 0; + ctx->W64[10] = 0; + ctx->W64[11] = 0; + ctx->W64[12] = 0; + ctx->W64[13] = 0; + ctx->W64[14] = 0; + ctx->W64[15] = swap64 ((u64) (final_len * 8)); + break; + } + } + + switch (ctx->dgst_len) + { + case BLSZ256: orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); + ctx->dgst32[ 0] = swap32 (ctx->dgst32[0]); + ctx->dgst32[ 1] = swap32 (ctx->dgst32[1]); + ctx->dgst32[ 2] = swap32 (ctx->dgst32[2]); + ctx->dgst32[ 3] = swap32 (ctx->dgst32[3]); + ctx->dgst32[ 4] = swap32 (ctx->dgst32[4]); + ctx->dgst32[ 5] = swap32 (ctx->dgst32[5]); + ctx->dgst32[ 6] = swap32 (ctx->dgst32[6]); + ctx->dgst32[ 7] = swap32 (ctx->dgst32[7]); + ctx->dgst32[ 8] = 0; + ctx->dgst32[ 9] = 0; + ctx->dgst32[10] = 0; + ctx->dgst32[11] = 0; + ctx->dgst32[12] = 0; + ctx->dgst32[13] = 0; + ctx->dgst32[14] = 0; + ctx->dgst32[15] = 0; + break; + case BLSZ384: orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + ctx->dgst64[0] = swap64 (ctx->dgst64[0]); + ctx->dgst64[1] = swap64 (ctx->dgst64[1]); + ctx->dgst64[2] = swap64 (ctx->dgst64[2]); + ctx->dgst64[3] = swap64 (ctx->dgst64[3]); + ctx->dgst64[4] = swap64 (ctx->dgst64[4]); + ctx->dgst64[5] = swap64 (ctx->dgst64[5]); + ctx->dgst64[6] = 0; + ctx->dgst64[7] = 0; + break; + case BLSZ512: orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); + ctx->dgst64[0] = swap64 (ctx->dgst64[0]); + ctx->dgst64[1] = swap64 (ctx->dgst64[1]); + ctx->dgst64[2] = swap64 (ctx->dgst64[2]); + ctx->dgst64[3] = swap64 (ctx->dgst64[3]); + ctx->dgst64[4] = swap64 (ctx->dgst64[4]); + ctx->dgst64[5] = swap64 (ctx->dgst64[5]); + ctx->dgst64[6] = swap64 (ctx->dgst64[6]); + ctx->dgst64[7] = swap64 (ctx->dgst64[7]); + break; + } + + return ex; +} + +__kernel void m10700_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + sha256_ctx_t ctx; + + sha256_init (&ctx); + + sha256_update_global_swap (&ctx, pws[gid].i, pws[gid].pw_len); + + sha256_update_global_swap (&ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); + + sha256_final (&ctx); + + tmps[gid].dgst32[0] = swap32_S (ctx.h[0]); + tmps[gid].dgst32[1] = swap32_S (ctx.h[1]); + tmps[gid].dgst32[2] = swap32_S (ctx.h[2]); + tmps[gid].dgst32[3] = swap32_S (ctx.h[3]); + tmps[gid].dgst32[4] = swap32_S (ctx.h[4]); + tmps[gid].dgst32[5] = swap32_S (ctx.h[5]); + tmps[gid].dgst32[6] = swap32_S (ctx.h[6]); + tmps[gid].dgst32[7] = swap32_S (ctx.h[7]); + tmps[gid].dgst_len = BLSZ256; + tmps[gid].W_len = WORDSZ256; +} + +__kernel void m10700_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + const u32 lsz = get_local_size (0); + + /** + * aes shared + */ + + #ifdef REAL_SHM + + __local u32 s_te0[256]; + __local u32 s_te1[256]; + __local u32 s_te2[256]; + __local u32 s_te3[256]; + __local u32 s_te4[256]; + + for (u32 i = lid; i < 256; i += lsz) + { + s_te0[i] = te0[i]; + s_te1[i] = te1[i]; + s_te2[i] = te2[i]; + s_te3[i] = te3[i]; + s_te4[i] = te4[i]; + } + + barrier (CLK_LOCAL_MEM_FENCE); + + #else + + __constant u32a *s_te0 = te0; + __constant u32a *s_te1 = te1; + __constant u32a *s_te2 = te2; + __constant u32a *s_te3 = te3; + __constant u32a *s_te4 = te4; + + #endif + + if (gid >= gid_max) return; + + /** + * base + */ + + u32 w0[4]; + + w0[0] = pws[gid].i[0]; + w0[1] = pws[gid].i[1]; + w0[2] = pws[gid].i[2]; + w0[3] = pws[gid].i[3]; + + const u32 pw_len = pws[gid].pw_len; + + if (pw_len == 0) return; + + /** + * digest + */ + + ctx_t ctx; + + ctx.dgst64[0] = tmps[gid].dgst64[0]; + ctx.dgst64[1] = tmps[gid].dgst64[1]; + ctx.dgst64[2] = tmps[gid].dgst64[2]; + ctx.dgst64[3] = tmps[gid].dgst64[3]; + ctx.dgst64[4] = tmps[gid].dgst64[4]; + ctx.dgst64[5] = tmps[gid].dgst64[5]; + ctx.dgst64[6] = tmps[gid].dgst64[6]; + ctx.dgst64[7] = tmps[gid].dgst64[7]; + ctx.dgst_len = tmps[gid].dgst_len; + ctx.W_len = tmps[gid].W_len; + + u32 ex = 0; + + for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) + { + ex = do_round (w0, pw_len, &ctx, s_te0, s_te1, s_te2, s_te3, s_te4); + } + + if ((loop_pos + loop_cnt) == 64) + { + for (u32 i = 64; i < (ex & 0xff) + 32; i++) + { + ex = do_round (w0, pw_len, &ctx, s_te0, s_te1, s_te2, s_te3, s_te4); + } + } + + tmps[gid].dgst64[0] = ctx.dgst64[0]; + tmps[gid].dgst64[1] = ctx.dgst64[1]; + tmps[gid].dgst64[2] = ctx.dgst64[2]; + tmps[gid].dgst64[3] = ctx.dgst64[3]; + tmps[gid].dgst64[4] = ctx.dgst64[4]; + tmps[gid].dgst64[5] = ctx.dgst64[5]; + tmps[gid].dgst64[6] = ctx.dgst64[6]; + tmps[gid].dgst64[7] = ctx.dgst64[7]; + tmps[gid].dgst_len = ctx.dgst_len; + tmps[gid].W_len = ctx.W_len; +} + +__kernel void m10700_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 r0 = swap32_S (tmps[gid].dgst32[DGST_R0]); + const u32 r1 = swap32_S (tmps[gid].dgst32[DGST_R1]); + const u32 r2 = swap32_S (tmps[gid].dgst32[DGST_R2]); + const u32 r3 = swap32_S (tmps[gid].dgst32[DGST_R3]); + + #define il_pos 0 + + #include COMPARE_M +} diff --git a/OpenCL/m10700-pure.cl b/OpenCL/m10700-pure.cl deleted file mode 100644 index a341d3d6e..000000000 --- a/OpenCL/m10700-pure.cl +++ /dev/null @@ -1,1257 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_hash_sha256.cl" -#include "inc_hash_sha384.cl" -#include "inc_hash_sha512.cl" -#include "inc_cipher_aes.cl" - -#define COMPARE_S "inc_comp_single.cl" -#define COMPARE_M "inc_comp_multi.cl" - -#define PUTCHAR(a,p,c) ((u8 *)(a))[(p)] = (u8) (c) -#define GETCHAR(a,p) ((u8 *)(a))[(p)] - -#define PUTCHAR_BE(a,p,c) ((u8 *)(a))[(p) ^ 3] = (u8) (c) -#define GETCHAR_BE(a,p) ((u8 *)(a))[(p) ^ 3] - -void aes128_encrypt_cbc (const u32 *aes_ks, u32 *aes_iv, const u32 *in, u32 *out, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 data[4]; - - data[0] = swap32_S (in[0]); - data[1] = swap32_S (in[1]); - data[2] = swap32_S (in[2]); - data[3] = swap32_S (in[3]); - - data[0] ^= aes_iv[0]; - data[1] ^= aes_iv[1]; - data[2] ^= aes_iv[2]; - data[3] ^= aes_iv[3]; - - aes128_encrypt (aes_ks, data, out, s_te0, s_te1, s_te2, s_te3, s_te4); - - aes_iv[0] = out[0]; - aes_iv[1] = out[1]; - aes_iv[2] = out[2]; - aes_iv[3] = out[3]; - - out[0] = swap32_S (out[0]); - out[1] = swap32_S (out[1]); - out[2] = swap32_S (out[2]); - out[3] = swap32_S (out[3]); -} - -u32 sha256_update_aes_64 (sha256_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 ex = 0; - - const int pos = ctx->len & 63; - - ctx->len += len; - - if ((pos + len) < 64) - { - switch_buffer_by_offset_be_S (w0, w1, w2, w3, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - } - else - { - u32 c0[4] = { 0 }; - u32 c1[4] = { 0 }; - u32 c2[4] = { 0 }; - u32 c3[4] = { 0 }; - - switch_buffer_by_offset_carry_be_S (w0, w1, w2, w3, c0, c1, c2, c3, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - - ex = ctx->w3[3] & 0xff; - - sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); - - ctx->w0[0] = c0[0]; - ctx->w0[1] = c0[1]; - ctx->w0[2] = c0[2]; - ctx->w0[3] = c0[3]; - ctx->w1[0] = c1[0]; - ctx->w1[1] = c1[1]; - ctx->w1[2] = c1[2]; - ctx->w1[3] = c1[3]; - ctx->w2[0] = c2[0]; - ctx->w2[1] = c2[1]; - ctx->w2[2] = c2[2]; - ctx->w2[3] = c2[3]; - ctx->w3[0] = c3[0]; - ctx->w3[1] = c3[1]; - ctx->w3[2] = c3[2]; - ctx->w3[3] = c3[3]; - } - - return ex; -} - -void sha256_update_aes (sha256_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - - int pos1; - int pos4; - - for (pos1 = 0, pos4 = 0; pos1 < len - 64; pos1 += 64, pos4 += 16) - { - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - - sha256_update_aes_64 (ctx, w0, w1, w2, w3, 64, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - - sha256_update_aes_64 (ctx, w0, w1, w2, w3, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); -} - -void sha256_final_aes (sha256_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - int pos = ctx->len & 63; - - // no encryption needed, because pos is always 0 - - append_0x80_4x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, pos ^ 3); - - if (pos >= 56) - { - sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); - - ctx->w0[0] = 0; - ctx->w0[1] = 0; - ctx->w0[2] = 0; - ctx->w0[3] = 0; - ctx->w1[0] = 0; - ctx->w1[1] = 0; - ctx->w1[2] = 0; - ctx->w1[3] = 0; - ctx->w2[0] = 0; - ctx->w2[1] = 0; - ctx->w2[2] = 0; - ctx->w2[3] = 0; - ctx->w3[0] = 0; - ctx->w3[1] = 0; - ctx->w3[2] = 0; - ctx->w3[3] = 0; - } - - ctx->w3[2] = 0; - ctx->w3[3] = ctx->len * 8; - - sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); -} - -void sha384_update_aes_128 (sha384_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], u32 w4[4], u32 w5[4], u32 w6[4], u32 w7[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - const int pos = ctx->len & 127; - - ctx->len += len; - - if ((pos + len) < 128) - { - switch_buffer_by_offset_8x4_be_S (w0, w1, w2, w3, w4, w5, w6, w7, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - ctx->w4[0] |= w4[0]; - ctx->w4[1] |= w4[1]; - ctx->w4[2] |= w4[2]; - ctx->w4[3] |= w4[3]; - ctx->w5[0] |= w5[0]; - ctx->w5[1] |= w5[1]; - ctx->w5[2] |= w5[2]; - ctx->w5[3] |= w5[3]; - ctx->w6[0] |= w6[0]; - ctx->w6[1] |= w6[1]; - ctx->w6[2] |= w6[2]; - ctx->w6[3] |= w6[3]; - ctx->w7[0] |= w7[0]; - ctx->w7[1] |= w7[1]; - ctx->w7[2] |= w7[2]; - ctx->w7[3] |= w7[3]; - } - else - { - u32 c0[4] = { 0 }; - u32 c1[4] = { 0 }; - u32 c2[4] = { 0 }; - u32 c3[4] = { 0 }; - u32 c4[4] = { 0 }; - u32 c5[4] = { 0 }; - u32 c6[4] = { 0 }; - u32 c7[4] = { 0 }; - - switch_buffer_by_offset_8x4_carry_be_S (w0, w1, w2, w3, w4, w5, w6, w7, c0, c1, c2, c3, c4, c5, c6, c7, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - ctx->w4[0] |= w4[0]; - ctx->w4[1] |= w4[1]; - ctx->w4[2] |= w4[2]; - ctx->w4[3] |= w4[3]; - ctx->w5[0] |= w5[0]; - ctx->w5[1] |= w5[1]; - ctx->w5[2] |= w5[2]; - ctx->w5[3] |= w5[3]; - ctx->w6[0] |= w6[0]; - ctx->w6[1] |= w6[1]; - ctx->w6[2] |= w6[2]; - ctx->w6[3] |= w6[3]; - ctx->w7[0] |= w7[0]; - ctx->w7[1] |= w7[1]; - ctx->w7[2] |= w7[2]; - ctx->w7[3] |= w7[3]; - - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w4, ctx->w4, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w5, ctx->w5, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w6, ctx->w6, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w7, ctx->w7, s_te0, s_te1, s_te2, s_te3, s_te4); - - sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); - - ctx->w0[0] = c0[0]; - ctx->w0[1] = c0[1]; - ctx->w0[2] = c0[2]; - ctx->w0[3] = c0[3]; - ctx->w1[0] = c1[0]; - ctx->w1[1] = c1[1]; - ctx->w1[2] = c1[2]; - ctx->w1[3] = c1[3]; - ctx->w2[0] = c2[0]; - ctx->w2[1] = c2[1]; - ctx->w2[2] = c2[2]; - ctx->w2[3] = c2[3]; - ctx->w3[0] = c3[0]; - ctx->w3[1] = c3[1]; - ctx->w3[2] = c3[2]; - ctx->w3[3] = c3[3]; - ctx->w4[0] = c4[0]; - ctx->w4[1] = c4[1]; - ctx->w4[2] = c4[2]; - ctx->w4[3] = c4[3]; - ctx->w5[0] = c5[0]; - ctx->w5[1] = c5[1]; - ctx->w5[2] = c5[2]; - ctx->w5[3] = c5[3]; - ctx->w6[0] = c6[0]; - ctx->w6[1] = c6[1]; - ctx->w6[2] = c6[2]; - ctx->w6[3] = c6[3]; - ctx->w7[0] = c7[0]; - ctx->w7[1] = c7[1]; - ctx->w7[2] = c7[2]; - ctx->w7[3] = c7[3]; - } -} - -void sha384_update_aes (sha384_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - u32 w4[4]; - u32 w5[4]; - u32 w6[4]; - u32 w7[4]; - - int pos1; - int pos4; - - for (pos1 = 0, pos4 = 0; pos1 < len - 128; pos1 += 128, pos4 += 32) - { - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - w4[0] = w[pos4 + 16]; - w4[1] = w[pos4 + 17]; - w4[2] = w[pos4 + 18]; - w4[3] = w[pos4 + 19]; - w5[0] = w[pos4 + 20]; - w5[1] = w[pos4 + 21]; - w5[2] = w[pos4 + 22]; - w5[3] = w[pos4 + 23]; - w6[0] = w[pos4 + 24]; - w6[1] = w[pos4 + 25]; - w6[2] = w[pos4 + 26]; - w6[3] = w[pos4 + 27]; - w7[0] = w[pos4 + 28]; - w7[1] = w[pos4 + 29]; - w7[2] = w[pos4 + 30]; - w7[3] = w[pos4 + 31]; - - sha384_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, 128, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - w4[0] = w[pos4 + 16]; - w4[1] = w[pos4 + 17]; - w4[2] = w[pos4 + 18]; - w4[3] = w[pos4 + 19]; - w5[0] = w[pos4 + 20]; - w5[1] = w[pos4 + 21]; - w5[2] = w[pos4 + 22]; - w5[3] = w[pos4 + 23]; - w6[0] = w[pos4 + 24]; - w6[1] = w[pos4 + 25]; - w6[2] = w[pos4 + 26]; - w6[3] = w[pos4 + 27]; - w7[0] = w[pos4 + 28]; - w7[1] = w[pos4 + 29]; - w7[2] = w[pos4 + 30]; - w7[3] = w[pos4 + 31]; - - sha384_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); -} - -void sha384_final_aes (sha384_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - int pos = ctx->len & 127; - - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - - append_0x80_8x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, pos ^ 3); - - if (pos >= 112) - { - sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); - - ctx->w0[0] = 0; - ctx->w0[1] = 0; - ctx->w0[2] = 0; - ctx->w0[3] = 0; - ctx->w1[0] = 0; - ctx->w1[1] = 0; - ctx->w1[2] = 0; - ctx->w1[3] = 0; - ctx->w2[0] = 0; - ctx->w2[1] = 0; - ctx->w2[2] = 0; - ctx->w2[3] = 0; - ctx->w3[0] = 0; - ctx->w3[1] = 0; - ctx->w3[2] = 0; - ctx->w3[3] = 0; - ctx->w4[0] = 0; - ctx->w4[1] = 0; - ctx->w4[2] = 0; - ctx->w4[3] = 0; - ctx->w5[0] = 0; - ctx->w5[1] = 0; - ctx->w5[2] = 0; - ctx->w5[3] = 0; - ctx->w6[0] = 0; - ctx->w6[1] = 0; - ctx->w6[2] = 0; - ctx->w6[3] = 0; - ctx->w7[0] = 0; - ctx->w7[1] = 0; - ctx->w7[2] = 0; - ctx->w7[3] = 0; - } - - ctx->w7[2] = 0; - ctx->w7[3] = ctx->len * 8; - - sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); -} - -void sha512_update_aes_128 (sha512_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], u32 w4[4], u32 w5[4], u32 w6[4], u32 w7[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - const int pos = ctx->len & 127; - - ctx->len += len; - - if ((pos + len) < 128) - { - switch_buffer_by_offset_8x4_be_S (w0, w1, w2, w3, w4, w5, w6, w7, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - ctx->w4[0] |= w4[0]; - ctx->w4[1] |= w4[1]; - ctx->w4[2] |= w4[2]; - ctx->w4[3] |= w4[3]; - ctx->w5[0] |= w5[0]; - ctx->w5[1] |= w5[1]; - ctx->w5[2] |= w5[2]; - ctx->w5[3] |= w5[3]; - ctx->w6[0] |= w6[0]; - ctx->w6[1] |= w6[1]; - ctx->w6[2] |= w6[2]; - ctx->w6[3] |= w6[3]; - ctx->w7[0] |= w7[0]; - ctx->w7[1] |= w7[1]; - ctx->w7[2] |= w7[2]; - ctx->w7[3] |= w7[3]; - } - else - { - u32 c0[4] = { 0 }; - u32 c1[4] = { 0 }; - u32 c2[4] = { 0 }; - u32 c3[4] = { 0 }; - u32 c4[4] = { 0 }; - u32 c5[4] = { 0 }; - u32 c6[4] = { 0 }; - u32 c7[4] = { 0 }; - - switch_buffer_by_offset_8x4_carry_be_S (w0, w1, w2, w3, w4, w5, w6, w7, c0, c1, c2, c3, c4, c5, c6, c7, pos); - - ctx->w0[0] |= w0[0]; - ctx->w0[1] |= w0[1]; - ctx->w0[2] |= w0[2]; - ctx->w0[3] |= w0[3]; - ctx->w1[0] |= w1[0]; - ctx->w1[1] |= w1[1]; - ctx->w1[2] |= w1[2]; - ctx->w1[3] |= w1[3]; - ctx->w2[0] |= w2[0]; - ctx->w2[1] |= w2[1]; - ctx->w2[2] |= w2[2]; - ctx->w2[3] |= w2[3]; - ctx->w3[0] |= w3[0]; - ctx->w3[1] |= w3[1]; - ctx->w3[2] |= w3[2]; - ctx->w3[3] |= w3[3]; - ctx->w4[0] |= w4[0]; - ctx->w4[1] |= w4[1]; - ctx->w4[2] |= w4[2]; - ctx->w4[3] |= w4[3]; - ctx->w5[0] |= w5[0]; - ctx->w5[1] |= w5[1]; - ctx->w5[2] |= w5[2]; - ctx->w5[3] |= w5[3]; - ctx->w6[0] |= w6[0]; - ctx->w6[1] |= w6[1]; - ctx->w6[2] |= w6[2]; - ctx->w6[3] |= w6[3]; - ctx->w7[0] |= w7[0]; - ctx->w7[1] |= w7[1]; - ctx->w7[2] |= w7[2]; - ctx->w7[3] |= w7[3]; - - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w4, ctx->w4, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w5, ctx->w5, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w6, ctx->w6, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w7, ctx->w7, s_te0, s_te1, s_te2, s_te3, s_te4); - - sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); - - ctx->w0[0] = c0[0]; - ctx->w0[1] = c0[1]; - ctx->w0[2] = c0[2]; - ctx->w0[3] = c0[3]; - ctx->w1[0] = c1[0]; - ctx->w1[1] = c1[1]; - ctx->w1[2] = c1[2]; - ctx->w1[3] = c1[3]; - ctx->w2[0] = c2[0]; - ctx->w2[1] = c2[1]; - ctx->w2[2] = c2[2]; - ctx->w2[3] = c2[3]; - ctx->w3[0] = c3[0]; - ctx->w3[1] = c3[1]; - ctx->w3[2] = c3[2]; - ctx->w3[3] = c3[3]; - ctx->w4[0] = c4[0]; - ctx->w4[1] = c4[1]; - ctx->w4[2] = c4[2]; - ctx->w4[3] = c4[3]; - ctx->w5[0] = c5[0]; - ctx->w5[1] = c5[1]; - ctx->w5[2] = c5[2]; - ctx->w5[3] = c5[3]; - ctx->w6[0] = c6[0]; - ctx->w6[1] = c6[1]; - ctx->w6[2] = c6[2]; - ctx->w6[3] = c6[3]; - ctx->w7[0] = c7[0]; - ctx->w7[1] = c7[1]; - ctx->w7[2] = c7[2]; - ctx->w7[3] = c7[3]; - } -} - -void sha512_update_aes (sha512_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - u32 w4[4]; - u32 w5[4]; - u32 w6[4]; - u32 w7[4]; - - int pos1; - int pos4; - - for (pos1 = 0, pos4 = 0; pos1 < len - 128; pos1 += 128, pos4 += 32) - { - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - w4[0] = w[pos4 + 16]; - w4[1] = w[pos4 + 17]; - w4[2] = w[pos4 + 18]; - w4[3] = w[pos4 + 19]; - w5[0] = w[pos4 + 20]; - w5[1] = w[pos4 + 21]; - w5[2] = w[pos4 + 22]; - w5[3] = w[pos4 + 23]; - w6[0] = w[pos4 + 24]; - w6[1] = w[pos4 + 25]; - w6[2] = w[pos4 + 26]; - w6[3] = w[pos4 + 27]; - w7[0] = w[pos4 + 28]; - w7[1] = w[pos4 + 29]; - w7[2] = w[pos4 + 30]; - w7[3] = w[pos4 + 31]; - - sha512_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, 128, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - w0[0] = w[pos4 + 0]; - w0[1] = w[pos4 + 1]; - w0[2] = w[pos4 + 2]; - w0[3] = w[pos4 + 3]; - w1[0] = w[pos4 + 4]; - w1[1] = w[pos4 + 5]; - w1[2] = w[pos4 + 6]; - w1[3] = w[pos4 + 7]; - w2[0] = w[pos4 + 8]; - w2[1] = w[pos4 + 9]; - w2[2] = w[pos4 + 10]; - w2[3] = w[pos4 + 11]; - w3[0] = w[pos4 + 12]; - w3[1] = w[pos4 + 13]; - w3[2] = w[pos4 + 14]; - w3[3] = w[pos4 + 15]; - w4[0] = w[pos4 + 16]; - w4[1] = w[pos4 + 17]; - w4[2] = w[pos4 + 18]; - w4[3] = w[pos4 + 19]; - w5[0] = w[pos4 + 20]; - w5[1] = w[pos4 + 21]; - w5[2] = w[pos4 + 22]; - w5[3] = w[pos4 + 23]; - w6[0] = w[pos4 + 24]; - w6[1] = w[pos4 + 25]; - w6[2] = w[pos4 + 26]; - w6[3] = w[pos4 + 27]; - w7[0] = w[pos4 + 28]; - w7[1] = w[pos4 + 29]; - w7[2] = w[pos4 + 30]; - w7[3] = w[pos4 + 31]; - - sha512_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); -} - -void sha512_final_aes (sha512_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - int pos = ctx->len & 127; - - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); - aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - - append_0x80_8x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, pos ^ 3); - - if (pos >= 112) - { - sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); - - ctx->w0[0] = 0; - ctx->w0[1] = 0; - ctx->w0[2] = 0; - ctx->w0[3] = 0; - ctx->w1[0] = 0; - ctx->w1[1] = 0; - ctx->w1[2] = 0; - ctx->w1[3] = 0; - ctx->w2[0] = 0; - ctx->w2[1] = 0; - ctx->w2[2] = 0; - ctx->w2[3] = 0; - ctx->w3[0] = 0; - ctx->w3[1] = 0; - ctx->w3[2] = 0; - ctx->w3[3] = 0; - ctx->w4[0] = 0; - ctx->w4[1] = 0; - ctx->w4[2] = 0; - ctx->w4[3] = 0; - ctx->w5[0] = 0; - ctx->w5[1] = 0; - ctx->w5[2] = 0; - ctx->w5[3] = 0; - ctx->w6[0] = 0; - ctx->w6[1] = 0; - ctx->w6[2] = 0; - ctx->w6[3] = 0; - ctx->w7[0] = 0; - ctx->w7[1] = 0; - ctx->w7[2] = 0; - ctx->w7[3] = 0; - } - - ctx->w7[2] = 0; - ctx->w7[3] = ctx->len * 8; - - sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); -} - -int find_sum (const u32 *w, const u32 pw_len, u32 *bb, const u32 *aes_ks, const u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - u32 data[4]; - - data[0] = w[0]; - data[1] = w[1]; - data[2] = w[2]; - data[3] = w[3]; - - for (int i = pw_len, j = 0; i < 16; i++, j++) - { - PUTCHAR_BE (data, i, GETCHAR_BE (bb, j)); - } - - data[0] = swap32_S (data[0]); - data[1] = swap32_S (data[1]); - data[2] = swap32_S (data[2]); - data[3] = swap32_S (data[3]); - - data[0] ^= aes_iv[0]; - data[1] ^= aes_iv[1]; - data[2] ^= aes_iv[2]; - data[3] ^= aes_iv[3]; - - u32 out[4]; - - aes128_encrypt (aes_ks, data, out, s_te0, s_te1, s_te2, s_te3, s_te4); - - u32 sum = 0; - - for (u32 i = 0; i < 4; i++) - { - sum += (out[i] >> 24) & 0xff; - sum += (out[i] >> 16) & 0xff; - sum += (out[i] >> 8) & 0xff; - sum += (out[i] >> 0) & 0xff; - } - - return sum; -} - -u32 do_round (const u32 *w, const u32 pw_len, pdf17l8_tmp_t *tmp, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - // get previous hash (already padded) - - u32 w0[4]; - u32 w1[4]; - u32 w2[4]; - u32 w3[4]; - u32 w4[4]; - u32 w5[4]; - u32 w6[4]; - u32 w7[4]; - - w0[0] = tmp->dgst32[0]; - w0[1] = tmp->dgst32[1]; - w0[2] = tmp->dgst32[2]; - w0[3] = tmp->dgst32[3]; - w1[0] = tmp->dgst32[4]; - w1[1] = tmp->dgst32[5]; - w1[2] = tmp->dgst32[6]; - w1[3] = tmp->dgst32[7]; - w2[0] = 0; - w2[1] = 0; - w2[2] = 0; - w2[3] = 0; - w3[0] = 0; - w3[1] = 0; - w3[2] = 0; - w3[3] = 0; - w4[0] = 0; - w4[1] = 0; - w4[2] = 0; - w4[3] = 0; - w5[0] = 0; - w5[1] = 0; - w5[2] = 0; - w5[3] = 0; - w6[0] = 0; - w6[1] = 0; - w6[2] = 0; - w6[3] = 0; - w7[0] = 0; - w7[1] = 0; - w7[2] = 0; - w7[3] = 0; - - // cipher setup - - u32 aes_key[4]; - - aes_key[0] = swap32_S (w0[0]); - aes_key[1] = swap32_S (w0[1]); - aes_key[2] = swap32_S (w0[2]); - aes_key[3] = swap32_S (w0[3]); - - u32 aes_ks[44]; - - aes128_set_encrypt_key (aes_ks, aes_key, s_te0, s_te1, s_te2, s_te3, s_te4); - - u32 aes_iv[4]; - - aes_iv[0] = swap32_S (w1[0]); - aes_iv[1] = swap32_S (w1[1]); - aes_iv[2] = swap32_S (w1[2]); - aes_iv[3] = swap32_S (w1[3]); - - // find hash to use - - const int sum = find_sum (w, pw_len, w0, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - int sum3 = sum % 3; - - // hash data - - u32 ex = 0; - - if (sum3 == 0) - { - sha256_ctx_t ctx256; - - sha256_init (&ctx256); - - for (int i = 0; i < 64; i++) - { - sha256_update_aes (&ctx256, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - w0[0] = tmp->dgst32[ 0]; - w0[1] = tmp->dgst32[ 1]; - w0[2] = tmp->dgst32[ 2]; - w0[3] = tmp->dgst32[ 3]; - w1[0] = tmp->dgst32[ 4]; - w1[1] = tmp->dgst32[ 5]; - w1[2] = tmp->dgst32[ 6]; - w1[3] = tmp->dgst32[ 7]; - w2[0] = tmp->dgst32[ 8]; - w2[1] = tmp->dgst32[ 9]; - w2[2] = tmp->dgst32[10]; - w2[3] = tmp->dgst32[11]; - w3[0] = tmp->dgst32[12]; - w3[1] = tmp->dgst32[13]; - w3[2] = tmp->dgst32[14]; - w3[3] = tmp->dgst32[15]; - w4[0] = 0; - w4[1] = 0; - w4[2] = 0; - w4[3] = 0; - w5[0] = 0; - w5[1] = 0; - w5[2] = 0; - w5[3] = 0; - w6[0] = 0; - w6[1] = 0; - w6[2] = 0; - w6[3] = 0; - w7[0] = 0; - w7[1] = 0; - w7[2] = 0; - w7[3] = 0; - - ex = sha256_update_aes_64 (&ctx256, w0, w1, w2, w3, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - sha256_final_aes (&ctx256, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - tmp->dgst32[ 0] = ctx256.h[0]; - tmp->dgst32[ 1] = ctx256.h[1]; - tmp->dgst32[ 2] = ctx256.h[2]; - tmp->dgst32[ 3] = ctx256.h[3]; - tmp->dgst32[ 4] = ctx256.h[4]; - tmp->dgst32[ 5] = ctx256.h[5]; - tmp->dgst32[ 6] = ctx256.h[6]; - tmp->dgst32[ 7] = ctx256.h[7]; - tmp->dgst32[ 8] = 0; - tmp->dgst32[ 9] = 0; - tmp->dgst32[10] = 0; - tmp->dgst32[11] = 0; - tmp->dgst32[12] = 0; - tmp->dgst32[13] = 0; - tmp->dgst32[14] = 0; - tmp->dgst32[15] = 0; - - tmp->dgst_len = 32; - } - else if (sum3 == 1) - { - sha384_ctx_t ctx384; - - sha384_init (&ctx384); - - for (int i = 0; i < 64; i++) - { - sha384_update_aes (&ctx384, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - w0[0] = tmp->dgst32[ 0]; - w0[1] = tmp->dgst32[ 1]; - w0[2] = tmp->dgst32[ 2]; - w0[3] = tmp->dgst32[ 3]; - w1[0] = tmp->dgst32[ 4]; - w1[1] = tmp->dgst32[ 5]; - w1[2] = tmp->dgst32[ 6]; - w1[3] = tmp->dgst32[ 7]; - w2[0] = tmp->dgst32[ 8]; - w2[1] = tmp->dgst32[ 9]; - w2[2] = tmp->dgst32[10]; - w2[3] = tmp->dgst32[11]; - w3[0] = tmp->dgst32[12]; - w3[1] = tmp->dgst32[13]; - w3[2] = tmp->dgst32[14]; - w3[3] = tmp->dgst32[15]; - w4[0] = 0; - w4[1] = 0; - w4[2] = 0; - w4[3] = 0; - w5[0] = 0; - w5[1] = 0; - w5[2] = 0; - w5[3] = 0; - w6[0] = 0; - w6[1] = 0; - w6[2] = 0; - w6[3] = 0; - w7[0] = 0; - w7[1] = 0; - w7[2] = 0; - w7[3] = 0; - - sha384_update_aes_128 (&ctx384, w0, w1, w2, w3, w4, w5, w6, w7, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - sha384_final_aes (&ctx384, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - ex = ctx384.w3[3] & 0xff; - - tmp->dgst32[ 0] = h32_from_64_S (ctx384.h[0]); - tmp->dgst32[ 1] = l32_from_64_S (ctx384.h[0]); - tmp->dgst32[ 2] = h32_from_64_S (ctx384.h[1]); - tmp->dgst32[ 3] = l32_from_64_S (ctx384.h[1]); - tmp->dgst32[ 4] = h32_from_64_S (ctx384.h[2]); - tmp->dgst32[ 5] = l32_from_64_S (ctx384.h[2]); - tmp->dgst32[ 6] = h32_from_64_S (ctx384.h[3]); - tmp->dgst32[ 7] = l32_from_64_S (ctx384.h[3]); - tmp->dgst32[ 8] = h32_from_64_S (ctx384.h[4]); - tmp->dgst32[ 9] = l32_from_64_S (ctx384.h[4]); - tmp->dgst32[10] = h32_from_64_S (ctx384.h[5]); - tmp->dgst32[11] = l32_from_64_S (ctx384.h[5]); - tmp->dgst32[12] = 0; - tmp->dgst32[13] = 0; - tmp->dgst32[14] = 0; - tmp->dgst32[15] = 0; - - tmp->dgst_len = 48; - } - else if (sum3 == 2) - { - sha512_ctx_t ctx512; - - sha512_init (&ctx512); - - for (int i = 0; i < 64; i++) - { - sha512_update_aes (&ctx512, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - w0[0] = tmp->dgst32[ 0]; - w0[1] = tmp->dgst32[ 1]; - w0[2] = tmp->dgst32[ 2]; - w0[3] = tmp->dgst32[ 3]; - w1[0] = tmp->dgst32[ 4]; - w1[1] = tmp->dgst32[ 5]; - w1[2] = tmp->dgst32[ 6]; - w1[3] = tmp->dgst32[ 7]; - w2[0] = tmp->dgst32[ 8]; - w2[1] = tmp->dgst32[ 9]; - w2[2] = tmp->dgst32[10]; - w2[3] = tmp->dgst32[11]; - w3[0] = tmp->dgst32[12]; - w3[1] = tmp->dgst32[13]; - w3[2] = tmp->dgst32[14]; - w3[3] = tmp->dgst32[15]; - w4[0] = 0; - w4[1] = 0; - w4[2] = 0; - w4[3] = 0; - w5[0] = 0; - w5[1] = 0; - w5[2] = 0; - w5[3] = 0; - w6[0] = 0; - w6[1] = 0; - w6[2] = 0; - w6[3] = 0; - w7[0] = 0; - w7[1] = 0; - w7[2] = 0; - w7[3] = 0; - - sha512_update_aes_128 (&ctx512, w0, w1, w2, w3, w4, w5, w6, w7, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - sha512_final_aes (&ctx512, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); - - ex = ctx512.w3[3] & 0xff; - - tmp->dgst32[ 0] = h32_from_64_S (ctx512.h[0]); - tmp->dgst32[ 1] = l32_from_64_S (ctx512.h[0]); - tmp->dgst32[ 2] = h32_from_64_S (ctx512.h[1]); - tmp->dgst32[ 3] = l32_from_64_S (ctx512.h[1]); - tmp->dgst32[ 4] = h32_from_64_S (ctx512.h[2]); - tmp->dgst32[ 5] = l32_from_64_S (ctx512.h[2]); - tmp->dgst32[ 6] = h32_from_64_S (ctx512.h[3]); - tmp->dgst32[ 7] = l32_from_64_S (ctx512.h[3]); - tmp->dgst32[ 8] = h32_from_64_S (ctx512.h[4]); - tmp->dgst32[ 9] = l32_from_64_S (ctx512.h[4]); - tmp->dgst32[10] = h32_from_64_S (ctx512.h[5]); - tmp->dgst32[11] = l32_from_64_S (ctx512.h[5]); - tmp->dgst32[12] = h32_from_64_S (ctx512.h[6]); - tmp->dgst32[13] = l32_from_64_S (ctx512.h[6]); - tmp->dgst32[14] = h32_from_64_S (ctx512.h[7]); - tmp->dgst32[15] = l32_from_64_S (ctx512.h[7]); - - tmp->dgst_len = 64; - } - - return ex; -} - -__kernel void m10700_init (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - sha256_ctx_t ctx; - - sha256_init (&ctx); - - sha256_update_global_swap (&ctx, pws[gid].i, pws[gid].pw_len); - - sha256_update_global_swap (&ctx, salt_bufs[salt_pos].salt_buf, salt_bufs[salt_pos].salt_len); - - sha256_final (&ctx); - - pdf17l8_tmp_t tmp; - - tmp.dgst32[ 0] = ctx.h[0]; - tmp.dgst32[ 1] = ctx.h[1]; - tmp.dgst32[ 2] = ctx.h[2]; - tmp.dgst32[ 3] = ctx.h[3]; - tmp.dgst32[ 4] = ctx.h[4]; - tmp.dgst32[ 5] = ctx.h[5]; - tmp.dgst32[ 6] = ctx.h[6]; - tmp.dgst32[ 7] = ctx.h[7]; - tmp.dgst32[ 8] = 0; - tmp.dgst32[ 9] = 0; - tmp.dgst32[10] = 0; - tmp.dgst32[11] = 0; - tmp.dgst32[12] = 0; - tmp.dgst32[13] = 0; - tmp.dgst32[14] = 0; - tmp.dgst32[15] = 0; - - tmp.dgst_len = 32; - - tmps[gid] = tmp; -} - -__kernel void m10700_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - const u32 gid = get_global_id (0); - const u32 lid = get_local_id (0); - const u32 lsz = get_local_size (0); - - /** - * aes shared - */ - - #ifdef REAL_SHM - - __local u32 s_te0[256]; - __local u32 s_te1[256]; - __local u32 s_te2[256]; - __local u32 s_te3[256]; - __local u32 s_te4[256]; - - for (u32 i = lid; i < 256; i += lsz) - { - s_te0[i] = te0[i]; - s_te1[i] = te1[i]; - s_te2[i] = te2[i]; - s_te3[i] = te3[i]; - s_te4[i] = te4[i]; - } - - barrier (CLK_LOCAL_MEM_FENCE); - - #else - - __constant u32a *s_te0 = te0; - __constant u32a *s_te1 = te1; - __constant u32a *s_te2 = te2; - __constant u32a *s_te3 = te3; - __constant u32a *s_te4 = te4; - - #endif - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * digest - */ - - pdf17l8_tmp_t tmp = tmps[gid]; - - u32 ex = 0; - - for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) - { - ex = do_round (w, pw_len, &tmp, s_te0, s_te1, s_te2, s_te3, s_te4); - } - - if ((loop_pos + loop_cnt) == 64) - { - for (u32 i = 64; i < (ex & 0xff) + 32; i++) - { - ex = do_round (w, pw_len, &tmp, s_te0, s_te1, s_te2, s_te3, s_te4); - } - } - - tmps[gid] = tmp; -} - -__kernel void m10700_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - const u32 lid = get_local_id (0); - - /** - * digest - */ - - const u32 r0 = tmps[gid].dgst32[DGST_R0]; - const u32 r1 = tmps[gid].dgst32[DGST_R1]; - const u32 r2 = tmps[gid].dgst32[DGST_R2]; - const u32 r3 = tmps[gid].dgst32[DGST_R3]; - - #define il_pos 0 - - #include COMPARE_M -} diff --git a/OpenCL/m10700.cl b/OpenCL/m10700.cl index 297b33917..a341d3d6e 100644 --- a/OpenCL/m10700.cl +++ b/OpenCL/m10700.cl @@ -16,652 +16,1089 @@ #define COMPARE_S "inc_comp_single.cl" #define COMPARE_M "inc_comp_multi.cl" -typedef struct +#define PUTCHAR(a,p,c) ((u8 *)(a))[(p)] = (u8) (c) +#define GETCHAR(a,p) ((u8 *)(a))[(p)] + +#define PUTCHAR_BE(a,p,c) ((u8 *)(a))[(p) ^ 3] = (u8) (c) +#define GETCHAR_BE(a,p) ((u8 *)(a))[(p) ^ 3] + +void aes128_encrypt_cbc (const u32 *aes_ks, u32 *aes_iv, const u32 *in, u32 *out, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) { - union - { - u32 dgst32[16]; - u64 dgst64[8]; - }; + u32 data[4]; - u32 dgst_len; + data[0] = swap32_S (in[0]); + data[1] = swap32_S (in[1]); + data[2] = swap32_S (in[2]); + data[3] = swap32_S (in[3]); - union - { - u32 W32[32]; - u64 W64[16]; - }; + data[0] ^= aes_iv[0]; + data[1] ^= aes_iv[1]; + data[2] ^= aes_iv[2]; + data[3] ^= aes_iv[3]; - u32 W_len; + aes128_encrypt (aes_ks, data, out, s_te0, s_te1, s_te2, s_te3, s_te4); -} ctx_t; + aes_iv[0] = out[0]; + aes_iv[1] = out[1]; + aes_iv[2] = out[2]; + aes_iv[3] = out[3]; -void orig_sha256_transform (const u32 *w0, const u32 *w1, const u32 *w2, const u32 *w3, u32 *digest) -{ - u32 a = digest[0]; - u32 b = digest[1]; - u32 c = digest[2]; - u32 d = digest[3]; - u32 e = digest[4]; - u32 f = digest[5]; - u32 g = digest[6]; - u32 h = digest[7]; - - u32 w0_t = swap32 (w0[0]); - u32 w1_t = swap32 (w0[1]); - u32 w2_t = swap32 (w0[2]); - u32 w3_t = swap32 (w0[3]); - u32 w4_t = swap32 (w1[0]); - u32 w5_t = swap32 (w1[1]); - u32 w6_t = swap32 (w1[2]); - u32 w7_t = swap32 (w1[3]); - u32 w8_t = swap32 (w2[0]); - u32 w9_t = swap32 (w2[1]); - u32 wa_t = swap32 (w2[2]); - u32 wb_t = swap32 (w2[3]); - u32 wc_t = swap32 (w3[0]); - u32 wd_t = swap32 (w3[1]); - u32 we_t = swap32 (w3[2]); - u32 wf_t = swap32 (w3[3]); - - #define ROUND256_EXPAND() \ - { \ - w0_t = SHA256_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA256_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA256_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA256_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA256_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA256_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA256_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA256_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA256_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA256_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA256_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA256_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA256_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA256_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA256_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA256_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND256_STEP(i) \ - { \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha256[i + 0]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha256[i + 1]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha256[i + 2]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha256[i + 3]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha256[i + 4]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha256[i + 5]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha256[i + 6]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha256[i + 7]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha256[i + 8]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha256[i + 9]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha256[i + 10]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha256[i + 11]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha256[i + 12]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha256[i + 13]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, c, d, e, f, g, h, a, b, we_t, k_sha256[i + 14]); \ - SHA256_STEP (SHA256_F0o, SHA256_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha256[i + 15]); \ - } - - ROUND256_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 64; i += 16) - { - ROUND256_EXPAND (); ROUND256_STEP (i); - } - - #undef ROUND256_EXPAND - #undef ROUND256_STEP - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; + out[0] = swap32_S (out[0]); + out[1] = swap32_S (out[1]); + out[2] = swap32_S (out[2]); + out[3] = swap32_S (out[3]); } -void orig_sha384_transform (const u64 *w0, const u64 *w1, const u64 *w2, const u64 *w3, u64 *digest) +u32 sha256_update_aes_64 (sha256_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) { - u64 a = digest[0]; - u64 b = digest[1]; - u64 c = digest[2]; - u64 d = digest[3]; - u64 e = digest[4]; - u64 f = digest[5]; - u64 g = digest[6]; - u64 h = digest[7]; + u32 ex = 0; - u64 w0_t = swap64 (w0[0]); - u64 w1_t = swap64 (w0[1]); - u64 w2_t = swap64 (w0[2]); - u64 w3_t = swap64 (w0[3]); - u64 w4_t = swap64 (w1[0]); - u64 w5_t = swap64 (w1[1]); - u64 w6_t = swap64 (w1[2]); - u64 w7_t = swap64 (w1[3]); - u64 w8_t = swap64 (w2[0]); - u64 w9_t = swap64 (w2[1]); - u64 wa_t = swap64 (w2[2]); - u64 wb_t = swap64 (w2[3]); - u64 wc_t = swap64 (w3[0]); - u64 wd_t = swap64 (w3[1]); - u64 we_t = swap64 (w3[2]); - u64 wf_t = swap64 (w3[3]); + const int pos = ctx->len & 63; - #define ROUND384_EXPAND() \ - { \ - w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } + ctx->len += len; - #define ROUND384_STEP(i) \ - { \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ - } - - ROUND384_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) + if ((pos + len) < 64) { - ROUND384_EXPAND (); ROUND384_STEP (i); - } + switch_buffer_by_offset_be_S (w0, w1, w2, w3, pos); - #undef ROUND384_EXPAND - #undef ROUND384_STEP - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -void orig_sha512_transform (const u64 *w0, const u64 *w1, const u64 *w2, const u64 *w3, u64 *digest) -{ - u64 a = digest[0]; - u64 b = digest[1]; - u64 c = digest[2]; - u64 d = digest[3]; - u64 e = digest[4]; - u64 f = digest[5]; - u64 g = digest[6]; - u64 h = digest[7]; - - u64 w0_t = swap64 (w0[0]); - u64 w1_t = swap64 (w0[1]); - u64 w2_t = swap64 (w0[2]); - u64 w3_t = swap64 (w0[3]); - u64 w4_t = swap64 (w1[0]); - u64 w5_t = swap64 (w1[1]); - u64 w6_t = swap64 (w1[2]); - u64 w7_t = swap64 (w1[3]); - u64 w8_t = swap64 (w2[0]); - u64 w9_t = swap64 (w2[1]); - u64 wa_t = swap64 (w2[2]); - u64 wb_t = swap64 (w2[3]); - u64 wc_t = swap64 (w3[0]); - u64 wd_t = swap64 (w3[1]); - u64 we_t = swap64 (w3[2]); - u64 wf_t = swap64 (w3[3]); - - #define ROUND512_EXPAND() \ - { \ - w0_t = SHA512_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA512_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA512_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA512_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA512_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA512_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA512_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA512_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA512_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA512_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA512_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA512_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA512_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA512_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA512_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA512_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND512_STEP(i) \ - { \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha512[i + 0]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha512[i + 1]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha512[i + 2]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha512[i + 3]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha512[i + 4]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha512[i + 5]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha512[i + 6]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha512[i + 7]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha512[i + 8]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha512[i + 9]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha512[i + 10]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha512[i + 11]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha512[i + 12]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha512[i + 13]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, c, d, e, f, g, h, a, b, we_t, k_sha512[i + 14]); \ - SHA512_STEP (SHA512_F0o, SHA512_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha512[i + 15]); \ - } - - ROUND512_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND512_EXPAND (); ROUND512_STEP (i); - } - - #undef ROUND512_EXPAND - #undef ROUND512_STEP - - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; -} - -#define AESSZ 16 // AES_BLOCK_SIZE - -#define BLSZ256 32 -#define BLSZ384 48 -#define BLSZ512 64 - -#define WORDSZ256 64 -#define WORDSZ384 128 -#define WORDSZ512 128 - -#define PWMAXSZ 32 // hashcat password length limit -#define BLMAXSZ BLSZ512 -#define WORDMAXSZ WORDSZ512 - -#define PWMAXSZ4 (PWMAXSZ / 4) -#define BLMAXSZ4 (BLMAXSZ / 4) -#define WORDMAXSZ4 (WORDMAXSZ / 4) -#define AESSZ4 (AESSZ / 4) - -void make_sc (u32 *sc, const u32 *pw, const u32 pw_len, const u32 *bl, const u32 bl_len) -{ - const u32 bd = bl_len / 4; - - const u32 pm = pw_len % 4; - const u32 pd = pw_len / 4; - - u32 idx = 0; - - if (pm == 0) - { - for (u32 i = 0; i < pd; i++) sc[idx++] = pw[i]; - for (u32 i = 0; i < bd; i++) sc[idx++] = bl[i]; - for (u32 i = 0; i < 4; i++) sc[idx++] = sc[i]; + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; } else { - u32 pm4 = 4 - pm; + u32 c0[4] = { 0 }; + u32 c1[4] = { 0 }; + u32 c2[4] = { 0 }; + u32 c3[4] = { 0 }; - u32 i; + switch_buffer_by_offset_carry_be_S (w0, w1, w2, w3, c0, c1, c2, c3, pos); - #if defined IS_AMD || defined IS_GENERIC - for (i = 0; i < pd; i++) sc[idx++] = pw[i]; - sc[idx++] = pw[i] - | amd_bytealign (bl[0], 0, pm4); - for (i = 1; i < bd; i++) sc[idx++] = amd_bytealign (bl[i], bl[i - 1], pm4); - sc[idx++] = amd_bytealign (sc[0], bl[i - 1], pm4); - for (i = 1; i < 4; i++) sc[idx++] = amd_bytealign (sc[i], sc[i - 1], pm4); - sc[idx++] = amd_bytealign ( 0, sc[i - 1], pm4); - #endif + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; - #ifdef IS_NV - int selector = (0x76543210 >> (pm4 * 4)) & 0xffff; + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - for (i = 0; i < pd; i++) sc[idx++] = pw[i]; - sc[idx++] = pw[i] - | __byte_perm ( 0, bl[0], selector); - for (i = 1; i < bd; i++) sc[idx++] = __byte_perm (bl[i - 1], bl[i], selector); - sc[idx++] = __byte_perm (bl[i - 1], sc[0], selector); - for (i = 1; i < 4; i++) sc[idx++] = __byte_perm (sc[i - 1], sc[i], selector); - sc[idx++] = __byte_perm (sc[i - 1], 0, selector); - #endif + ex = ctx->w3[3] & 0xff; + + sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); + + ctx->w0[0] = c0[0]; + ctx->w0[1] = c0[1]; + ctx->w0[2] = c0[2]; + ctx->w0[3] = c0[3]; + ctx->w1[0] = c1[0]; + ctx->w1[1] = c1[1]; + ctx->w1[2] = c1[2]; + ctx->w1[3] = c1[3]; + ctx->w2[0] = c2[0]; + ctx->w2[1] = c2[1]; + ctx->w2[2] = c2[2]; + ctx->w2[3] = c2[3]; + ctx->w3[0] = c3[0]; + ctx->w3[1] = c3[1]; + ctx->w3[2] = c3[2]; + ctx->w3[3] = c3[3]; } + + return ex; } -void make_pt_with_offset (u32 *pt, const u32 offset, const u32 *sc, const u32 pwbl_len) +void sha256_update_aes (sha256_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) { - const u32 m = offset % pwbl_len; + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; - const u32 om = m % 4; - const u32 od = m / 4; + int pos1; + int pos4; - #if defined IS_AMD || defined IS_GENERIC - pt[0] = amd_bytealign (sc[od + 1], sc[od + 0], om); - pt[1] = amd_bytealign (sc[od + 2], sc[od + 1], om); - pt[2] = amd_bytealign (sc[od + 3], sc[od + 2], om); - pt[3] = amd_bytealign (sc[od + 4], sc[od + 3], om); - #endif - - #ifdef IS_NV - int selector = (0x76543210 >> (om * 4)) & 0xffff; - - pt[0] = __byte_perm (sc[od + 0], sc[od + 1], selector); - pt[1] = __byte_perm (sc[od + 1], sc[od + 2], selector); - pt[2] = __byte_perm (sc[od + 2], sc[od + 3], selector); - pt[3] = __byte_perm (sc[od + 3], sc[od + 4], selector); - #endif -} - -void make_w_with_offset (ctx_t *ctx, const u32 W_len, const u32 offset, const u32 *sc, const u32 pwbl_len, u32 *iv, const u32 *ks, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) -{ - for (u32 k = 0, wk = 0; k < W_len; k += AESSZ, wk += AESSZ4) + for (pos1 = 0, pos4 = 0; pos1 < len - 64; pos1 += 64, pos4 += 16) { - u32 pt[AESSZ4]; + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; - make_pt_with_offset (pt, offset + k, sc, pwbl_len); + sha256_update_aes_64 (ctx, w0, w1, w2, w3, 64, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } - pt[0] ^= iv[0]; - pt[1] ^= iv[1]; - pt[2] ^= iv[2]; - pt[3] ^= iv[3]; + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; - aes128_encrypt (ks, pt, iv, s_te0, s_te1, s_te2, s_te3, s_te4); + sha256_update_aes_64 (ctx, w0, w1, w2, w3, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); +} - ctx->W32[wk + 0] = iv[0]; - ctx->W32[wk + 1] = iv[1]; - ctx->W32[wk + 2] = iv[2]; - ctx->W32[wk + 3] = iv[3]; +void sha256_final_aes (sha256_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + int pos = ctx->len & 63; + + // no encryption needed, because pos is always 0 + + append_0x80_4x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, pos ^ 3); + + if (pos >= 56) + { + sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); + + ctx->w0[0] = 0; + ctx->w0[1] = 0; + ctx->w0[2] = 0; + ctx->w0[3] = 0; + ctx->w1[0] = 0; + ctx->w1[1] = 0; + ctx->w1[2] = 0; + ctx->w1[3] = 0; + ctx->w2[0] = 0; + ctx->w2[1] = 0; + ctx->w2[2] = 0; + ctx->w2[3] = 0; + ctx->w3[0] = 0; + ctx->w3[1] = 0; + ctx->w3[2] = 0; + ctx->w3[3] = 0; + } + + ctx->w3[2] = 0; + ctx->w3[3] = ctx->len * 8; + + sha256_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->h); +} + +void sha384_update_aes_128 (sha384_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], u32 w4[4], u32 w5[4], u32 w6[4], u32 w7[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + const int pos = ctx->len & 127; + + ctx->len += len; + + if ((pos + len) < 128) + { + switch_buffer_by_offset_8x4_be_S (w0, w1, w2, w3, w4, w5, w6, w7, pos); + + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; + ctx->w4[0] |= w4[0]; + ctx->w4[1] |= w4[1]; + ctx->w4[2] |= w4[2]; + ctx->w4[3] |= w4[3]; + ctx->w5[0] |= w5[0]; + ctx->w5[1] |= w5[1]; + ctx->w5[2] |= w5[2]; + ctx->w5[3] |= w5[3]; + ctx->w6[0] |= w6[0]; + ctx->w6[1] |= w6[1]; + ctx->w6[2] |= w6[2]; + ctx->w6[3] |= w6[3]; + ctx->w7[0] |= w7[0]; + ctx->w7[1] |= w7[1]; + ctx->w7[2] |= w7[2]; + ctx->w7[3] |= w7[3]; + } + else + { + u32 c0[4] = { 0 }; + u32 c1[4] = { 0 }; + u32 c2[4] = { 0 }; + u32 c3[4] = { 0 }; + u32 c4[4] = { 0 }; + u32 c5[4] = { 0 }; + u32 c6[4] = { 0 }; + u32 c7[4] = { 0 }; + + switch_buffer_by_offset_8x4_carry_be_S (w0, w1, w2, w3, w4, w5, w6, w7, c0, c1, c2, c3, c4, c5, c6, c7, pos); + + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; + ctx->w4[0] |= w4[0]; + ctx->w4[1] |= w4[1]; + ctx->w4[2] |= w4[2]; + ctx->w4[3] |= w4[3]; + ctx->w5[0] |= w5[0]; + ctx->w5[1] |= w5[1]; + ctx->w5[2] |= w5[2]; + ctx->w5[3] |= w5[3]; + ctx->w6[0] |= w6[0]; + ctx->w6[1] |= w6[1]; + ctx->w6[2] |= w6[2]; + ctx->w6[3] |= w6[3]; + ctx->w7[0] |= w7[0]; + ctx->w7[1] |= w7[1]; + ctx->w7[2] |= w7[2]; + ctx->w7[3] |= w7[3]; + + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w4, ctx->w4, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w5, ctx->w5, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w6, ctx->w6, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w7, ctx->w7, s_te0, s_te1, s_te2, s_te3, s_te4); + + sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); + + ctx->w0[0] = c0[0]; + ctx->w0[1] = c0[1]; + ctx->w0[2] = c0[2]; + ctx->w0[3] = c0[3]; + ctx->w1[0] = c1[0]; + ctx->w1[1] = c1[1]; + ctx->w1[2] = c1[2]; + ctx->w1[3] = c1[3]; + ctx->w2[0] = c2[0]; + ctx->w2[1] = c2[1]; + ctx->w2[2] = c2[2]; + ctx->w2[3] = c2[3]; + ctx->w3[0] = c3[0]; + ctx->w3[1] = c3[1]; + ctx->w3[2] = c3[2]; + ctx->w3[3] = c3[3]; + ctx->w4[0] = c4[0]; + ctx->w4[1] = c4[1]; + ctx->w4[2] = c4[2]; + ctx->w4[3] = c4[3]; + ctx->w5[0] = c5[0]; + ctx->w5[1] = c5[1]; + ctx->w5[2] = c5[2]; + ctx->w5[3] = c5[3]; + ctx->w6[0] = c6[0]; + ctx->w6[1] = c6[1]; + ctx->w6[2] = c6[2]; + ctx->w6[3] = c6[3]; + ctx->w7[0] = c7[0]; + ctx->w7[1] = c7[1]; + ctx->w7[2] = c7[2]; + ctx->w7[3] = c7[3]; } } -u32 do_round (const u32 *pw, const u32 pw_len, ctx_t *ctx, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +void sha384_update_aes (sha384_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) { - // make scratch buffer + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; + u32 w4[4]; + u32 w5[4]; + u32 w6[4]; + u32 w7[4]; - u32 sc[PWMAXSZ4 + BLMAXSZ4 + AESSZ4]; + int pos1; + int pos4; - make_sc (sc, pw, pw_len, ctx->dgst32, ctx->dgst_len); + for (pos1 = 0, pos4 = 0; pos1 < len - 128; pos1 += 128, pos4 += 32) + { + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; + w4[0] = w[pos4 + 16]; + w4[1] = w[pos4 + 17]; + w4[2] = w[pos4 + 18]; + w4[3] = w[pos4 + 19]; + w5[0] = w[pos4 + 20]; + w5[1] = w[pos4 + 21]; + w5[2] = w[pos4 + 22]; + w5[3] = w[pos4 + 23]; + w6[0] = w[pos4 + 24]; + w6[1] = w[pos4 + 25]; + w6[2] = w[pos4 + 26]; + w6[3] = w[pos4 + 27]; + w7[0] = w[pos4 + 28]; + w7[1] = w[pos4 + 29]; + w7[2] = w[pos4 + 30]; + w7[3] = w[pos4 + 31]; - // make sure pwbl_len is calculcated before it gets changed + sha384_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, 128, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } - const u32 pwbl_len = pw_len + ctx->dgst_len; + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; + w4[0] = w[pos4 + 16]; + w4[1] = w[pos4 + 17]; + w4[2] = w[pos4 + 18]; + w4[3] = w[pos4 + 19]; + w5[0] = w[pos4 + 20]; + w5[1] = w[pos4 + 21]; + w5[2] = w[pos4 + 22]; + w5[3] = w[pos4 + 23]; + w6[0] = w[pos4 + 24]; + w6[1] = w[pos4 + 25]; + w6[2] = w[pos4 + 26]; + w6[3] = w[pos4 + 27]; + w7[0] = w[pos4 + 28]; + w7[1] = w[pos4 + 29]; + w7[2] = w[pos4 + 30]; + w7[3] = w[pos4 + 31]; - // init iv + sha384_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); +} - u32 iv[AESSZ4]; +void sha384_final_aes (sha384_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + int pos = ctx->len & 127; - iv[0] = ctx->dgst32[4]; - iv[1] = ctx->dgst32[5]; - iv[2] = ctx->dgst32[6]; - iv[3] = ctx->dgst32[7]; + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); - // init aes + append_0x80_8x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, pos ^ 3); - u32 ks[44]; + if (pos >= 112) + { + sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); - aes128_set_encrypt_key (ks, ctx->dgst32, s_te0, s_te1, s_te2, s_te3, s_te4); + ctx->w0[0] = 0; + ctx->w0[1] = 0; + ctx->w0[2] = 0; + ctx->w0[3] = 0; + ctx->w1[0] = 0; + ctx->w1[1] = 0; + ctx->w1[2] = 0; + ctx->w1[3] = 0; + ctx->w2[0] = 0; + ctx->w2[1] = 0; + ctx->w2[2] = 0; + ctx->w2[3] = 0; + ctx->w3[0] = 0; + ctx->w3[1] = 0; + ctx->w3[2] = 0; + ctx->w3[3] = 0; + ctx->w4[0] = 0; + ctx->w4[1] = 0; + ctx->w4[2] = 0; + ctx->w4[3] = 0; + ctx->w5[0] = 0; + ctx->w5[1] = 0; + ctx->w5[2] = 0; + ctx->w5[3] = 0; + ctx->w6[0] = 0; + ctx->w6[1] = 0; + ctx->w6[2] = 0; + ctx->w6[3] = 0; + ctx->w7[0] = 0; + ctx->w7[1] = 0; + ctx->w7[2] = 0; + ctx->w7[3] = 0; + } - // first call is special as the hash depends on the result of it - // but since we do not know about the outcome at this time - // we must use the max + ctx->w7[2] = 0; + ctx->w7[3] = ctx->len * 8; - make_w_with_offset (ctx, WORDMAXSZ, 0, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + sha384_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); +} - // now we can find out hash to use +void sha512_update_aes_128 (sha512_ctx_t *ctx, u32 w0[4], u32 w1[4], u32 w2[4], u32 w3[4], u32 w4[4], u32 w5[4], u32 w6[4], u32 w7[4], const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + const int pos = ctx->len & 127; + + ctx->len += len; + + if ((pos + len) < 128) + { + switch_buffer_by_offset_8x4_be_S (w0, w1, w2, w3, w4, w5, w6, w7, pos); + + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; + ctx->w4[0] |= w4[0]; + ctx->w4[1] |= w4[1]; + ctx->w4[2] |= w4[2]; + ctx->w4[3] |= w4[3]; + ctx->w5[0] |= w5[0]; + ctx->w5[1] |= w5[1]; + ctx->w5[2] |= w5[2]; + ctx->w5[3] |= w5[3]; + ctx->w6[0] |= w6[0]; + ctx->w6[1] |= w6[1]; + ctx->w6[2] |= w6[2]; + ctx->w6[3] |= w6[3]; + ctx->w7[0] |= w7[0]; + ctx->w7[1] |= w7[1]; + ctx->w7[2] |= w7[2]; + ctx->w7[3] |= w7[3]; + } + else + { + u32 c0[4] = { 0 }; + u32 c1[4] = { 0 }; + u32 c2[4] = { 0 }; + u32 c3[4] = { 0 }; + u32 c4[4] = { 0 }; + u32 c5[4] = { 0 }; + u32 c6[4] = { 0 }; + u32 c7[4] = { 0 }; + + switch_buffer_by_offset_8x4_carry_be_S (w0, w1, w2, w3, w4, w5, w6, w7, c0, c1, c2, c3, c4, c5, c6, c7, pos); + + ctx->w0[0] |= w0[0]; + ctx->w0[1] |= w0[1]; + ctx->w0[2] |= w0[2]; + ctx->w0[3] |= w0[3]; + ctx->w1[0] |= w1[0]; + ctx->w1[1] |= w1[1]; + ctx->w1[2] |= w1[2]; + ctx->w1[3] |= w1[3]; + ctx->w2[0] |= w2[0]; + ctx->w2[1] |= w2[1]; + ctx->w2[2] |= w2[2]; + ctx->w2[3] |= w2[3]; + ctx->w3[0] |= w3[0]; + ctx->w3[1] |= w3[1]; + ctx->w3[2] |= w3[2]; + ctx->w3[3] |= w3[3]; + ctx->w4[0] |= w4[0]; + ctx->w4[1] |= w4[1]; + ctx->w4[2] |= w4[2]; + ctx->w4[3] |= w4[3]; + ctx->w5[0] |= w5[0]; + ctx->w5[1] |= w5[1]; + ctx->w5[2] |= w5[2]; + ctx->w5[3] |= w5[3]; + ctx->w6[0] |= w6[0]; + ctx->w6[1] |= w6[1]; + ctx->w6[2] |= w6[2]; + ctx->w6[3] |= w6[3]; + ctx->w7[0] |= w7[0]; + ctx->w7[1] |= w7[1]; + ctx->w7[2] |= w7[2]; + ctx->w7[3] |= w7[3]; + + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w4, ctx->w4, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w5, ctx->w5, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w6, ctx->w6, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w7, ctx->w7, s_te0, s_te1, s_te2, s_te3, s_te4); + + sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); + + ctx->w0[0] = c0[0]; + ctx->w0[1] = c0[1]; + ctx->w0[2] = c0[2]; + ctx->w0[3] = c0[3]; + ctx->w1[0] = c1[0]; + ctx->w1[1] = c1[1]; + ctx->w1[2] = c1[2]; + ctx->w1[3] = c1[3]; + ctx->w2[0] = c2[0]; + ctx->w2[1] = c2[1]; + ctx->w2[2] = c2[2]; + ctx->w2[3] = c2[3]; + ctx->w3[0] = c3[0]; + ctx->w3[1] = c3[1]; + ctx->w3[2] = c3[2]; + ctx->w3[3] = c3[3]; + ctx->w4[0] = c4[0]; + ctx->w4[1] = c4[1]; + ctx->w4[2] = c4[2]; + ctx->w4[3] = c4[3]; + ctx->w5[0] = c5[0]; + ctx->w5[1] = c5[1]; + ctx->w5[2] = c5[2]; + ctx->w5[3] = c5[3]; + ctx->w6[0] = c6[0]; + ctx->w6[1] = c6[1]; + ctx->w6[2] = c6[2]; + ctx->w6[3] = c6[3]; + ctx->w7[0] = c7[0]; + ctx->w7[1] = c7[1]; + ctx->w7[2] = c7[2]; + ctx->w7[3] = c7[3]; + } +} + +void sha512_update_aes (sha512_ctx_t *ctx, const u32 *w, const int len, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; + u32 w4[4]; + u32 w5[4]; + u32 w6[4]; + u32 w7[4]; + + int pos1; + int pos4; + + for (pos1 = 0, pos4 = 0; pos1 < len - 128; pos1 += 128, pos4 += 32) + { + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; + w4[0] = w[pos4 + 16]; + w4[1] = w[pos4 + 17]; + w4[2] = w[pos4 + 18]; + w4[3] = w[pos4 + 19]; + w5[0] = w[pos4 + 20]; + w5[1] = w[pos4 + 21]; + w5[2] = w[pos4 + 22]; + w5[3] = w[pos4 + 23]; + w6[0] = w[pos4 + 24]; + w6[1] = w[pos4 + 25]; + w6[2] = w[pos4 + 26]; + w6[3] = w[pos4 + 27]; + w7[0] = w[pos4 + 28]; + w7[1] = w[pos4 + 29]; + w7[2] = w[pos4 + 30]; + w7[3] = w[pos4 + 31]; + + sha512_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, 128, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } + + w0[0] = w[pos4 + 0]; + w0[1] = w[pos4 + 1]; + w0[2] = w[pos4 + 2]; + w0[3] = w[pos4 + 3]; + w1[0] = w[pos4 + 4]; + w1[1] = w[pos4 + 5]; + w1[2] = w[pos4 + 6]; + w1[3] = w[pos4 + 7]; + w2[0] = w[pos4 + 8]; + w2[1] = w[pos4 + 9]; + w2[2] = w[pos4 + 10]; + w2[3] = w[pos4 + 11]; + w3[0] = w[pos4 + 12]; + w3[1] = w[pos4 + 13]; + w3[2] = w[pos4 + 14]; + w3[3] = w[pos4 + 15]; + w4[0] = w[pos4 + 16]; + w4[1] = w[pos4 + 17]; + w4[2] = w[pos4 + 18]; + w4[3] = w[pos4 + 19]; + w5[0] = w[pos4 + 20]; + w5[1] = w[pos4 + 21]; + w5[2] = w[pos4 + 22]; + w5[3] = w[pos4 + 23]; + w6[0] = w[pos4 + 24]; + w6[1] = w[pos4 + 25]; + w6[2] = w[pos4 + 26]; + w6[3] = w[pos4 + 27]; + w7[0] = w[pos4 + 28]; + w7[1] = w[pos4 + 29]; + w7[2] = w[pos4 + 30]; + w7[3] = w[pos4 + 31]; + + sha512_update_aes_128 (ctx, w0, w1, w2, w3, w4, w5, w6, w7, len - pos1, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); +} + +void sha512_final_aes (sha512_ctx_t *ctx, const u32 *aes_ks, u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + int pos = ctx->len & 127; + + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w0, ctx->w0, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w1, ctx->w1, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w2, ctx->w2, s_te0, s_te1, s_te2, s_te3, s_te4); + aes128_encrypt_cbc (aes_ks, aes_iv, ctx->w3, ctx->w3, s_te0, s_te1, s_te2, s_te3, s_te4); + + append_0x80_8x4_S (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, pos ^ 3); + + if (pos >= 112) + { + sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); + + ctx->w0[0] = 0; + ctx->w0[1] = 0; + ctx->w0[2] = 0; + ctx->w0[3] = 0; + ctx->w1[0] = 0; + ctx->w1[1] = 0; + ctx->w1[2] = 0; + ctx->w1[3] = 0; + ctx->w2[0] = 0; + ctx->w2[1] = 0; + ctx->w2[2] = 0; + ctx->w2[3] = 0; + ctx->w3[0] = 0; + ctx->w3[1] = 0; + ctx->w3[2] = 0; + ctx->w3[3] = 0; + ctx->w4[0] = 0; + ctx->w4[1] = 0; + ctx->w4[2] = 0; + ctx->w4[3] = 0; + ctx->w5[0] = 0; + ctx->w5[1] = 0; + ctx->w5[2] = 0; + ctx->w5[3] = 0; + ctx->w6[0] = 0; + ctx->w6[1] = 0; + ctx->w6[2] = 0; + ctx->w6[3] = 0; + ctx->w7[0] = 0; + ctx->w7[1] = 0; + ctx->w7[2] = 0; + ctx->w7[3] = 0; + } + + ctx->w7[2] = 0; + ctx->w7[3] = ctx->len * 8; + + sha512_transform (ctx->w0, ctx->w1, ctx->w2, ctx->w3, ctx->w4, ctx->w5, ctx->w6, ctx->w7, ctx->h); +} + +int find_sum (const u32 *w, const u32 pw_len, u32 *bb, const u32 *aes_ks, const u32 *aes_iv, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + u32 data[4]; + + data[0] = w[0]; + data[1] = w[1]; + data[2] = w[2]; + data[3] = w[3]; + + for (int i = pw_len, j = 0; i < 16; i++, j++) + { + PUTCHAR_BE (data, i, GETCHAR_BE (bb, j)); + } + + data[0] = swap32_S (data[0]); + data[1] = swap32_S (data[1]); + data[2] = swap32_S (data[2]); + data[3] = swap32_S (data[3]); + + data[0] ^= aes_iv[0]; + data[1] ^= aes_iv[1]; + data[2] ^= aes_iv[2]; + data[3] ^= aes_iv[3]; + + u32 out[4]; + + aes128_encrypt (aes_ks, data, out, s_te0, s_te1, s_te2, s_te3, s_te4); u32 sum = 0; for (u32 i = 0; i < 4; i++) { - sum += (ctx->W32[i] >> 24) & 0xff; - sum += (ctx->W32[i] >> 16) & 0xff; - sum += (ctx->W32[i] >> 8) & 0xff; - sum += (ctx->W32[i] >> 0) & 0xff; + sum += (out[i] >> 24) & 0xff; + sum += (out[i] >> 16) & 0xff; + sum += (out[i] >> 8) & 0xff; + sum += (out[i] >> 0) & 0xff; } - // init hash + return sum; +} - switch (sum % 3) - { - case 0: ctx->dgst32[0] = SHA256M_A; - ctx->dgst32[1] = SHA256M_B; - ctx->dgst32[2] = SHA256M_C; - ctx->dgst32[3] = SHA256M_D; - ctx->dgst32[4] = SHA256M_E; - ctx->dgst32[5] = SHA256M_F; - ctx->dgst32[6] = SHA256M_G; - ctx->dgst32[7] = SHA256M_H; - ctx->dgst_len = BLSZ256; - ctx->W_len = WORDSZ256; - orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); - orig_sha256_transform (&ctx->W32[16], &ctx->W32[20], &ctx->W32[24], &ctx->W32[28], ctx->dgst32); - break; - case 1: ctx->dgst64[0] = SHA384M_A; - ctx->dgst64[1] = SHA384M_B; - ctx->dgst64[2] = SHA384M_C; - ctx->dgst64[3] = SHA384M_D; - ctx->dgst64[4] = SHA384M_E; - ctx->dgst64[5] = SHA384M_F; - ctx->dgst64[6] = SHA384M_G; - ctx->dgst64[7] = SHA384M_H; - ctx->dgst_len = BLSZ384; - ctx->W_len = WORDSZ384; - orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - break; - case 2: ctx->dgst64[0] = SHA512M_A; - ctx->dgst64[1] = SHA512M_B; - ctx->dgst64[2] = SHA512M_C; - ctx->dgst64[3] = SHA512M_D; - ctx->dgst64[4] = SHA512M_E; - ctx->dgst64[5] = SHA512M_F; - ctx->dgst64[6] = SHA512M_G; - ctx->dgst64[7] = SHA512M_H; - ctx->dgst_len = BLSZ512; - ctx->W_len = WORDSZ512; - orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - break; - } +u32 do_round (const u32 *w, const u32 pw_len, pdf17l8_tmp_t *tmp, SHM_TYPE u32 *s_te0, SHM_TYPE u32 *s_te1, SHM_TYPE u32 *s_te2, SHM_TYPE u32 *s_te3, SHM_TYPE u32 *s_te4) +{ + // get previous hash (already padded) - // main loop + u32 w0[4]; + u32 w1[4]; + u32 w2[4]; + u32 w3[4]; + u32 w4[4]; + u32 w5[4]; + u32 w6[4]; + u32 w7[4]; - const u32 final_len = pwbl_len * 64; + w0[0] = tmp->dgst32[0]; + w0[1] = tmp->dgst32[1]; + w0[2] = tmp->dgst32[2]; + w0[3] = tmp->dgst32[3]; + w1[0] = tmp->dgst32[4]; + w1[1] = tmp->dgst32[5]; + w1[2] = tmp->dgst32[6]; + w1[3] = tmp->dgst32[7]; + w2[0] = 0; + w2[1] = 0; + w2[2] = 0; + w2[3] = 0; + w3[0] = 0; + w3[1] = 0; + w3[2] = 0; + w3[3] = 0; + w4[0] = 0; + w4[1] = 0; + w4[2] = 0; + w4[3] = 0; + w5[0] = 0; + w5[1] = 0; + w5[2] = 0; + w5[3] = 0; + w6[0] = 0; + w6[1] = 0; + w6[2] = 0; + w6[3] = 0; + w7[0] = 0; + w7[1] = 0; + w7[2] = 0; + w7[3] = 0; - const u32 iter_max = ctx->W_len - (ctx->W_len / 8); + // cipher setup - u32 offset; - u32 left; + u32 aes_key[4]; - for (offset = WORDMAXSZ, left = final_len - offset; left >= iter_max; offset += ctx->W_len, left -= ctx->W_len) - { - make_w_with_offset (ctx, ctx->W_len, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); + aes_key[0] = swap32_S (w0[0]); + aes_key[1] = swap32_S (w0[1]); + aes_key[2] = swap32_S (w0[2]); + aes_key[3] = swap32_S (w0[3]); - switch (ctx->dgst_len) - { - case BLSZ256: orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); - break; - case BLSZ384: orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - break; - case BLSZ512: orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - break; - } - } + u32 aes_ks[44]; + + aes128_set_encrypt_key (aes_ks, aes_key, s_te0, s_te1, s_te2, s_te3, s_te4); + + u32 aes_iv[4]; + + aes_iv[0] = swap32_S (w1[0]); + aes_iv[1] = swap32_S (w1[1]); + aes_iv[2] = swap32_S (w1[2]); + aes_iv[3] = swap32_S (w1[3]); + + // find hash to use + + const int sum = find_sum (w, pw_len, w0, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + int sum3 = sum % 3; + + // hash data u32 ex = 0; - if (left) + if (sum3 == 0) { - switch (ctx->dgst_len) - { - case BLSZ384: make_w_with_offset (ctx, 64, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); - ctx->W64[ 8] = 0x80; - ctx->W64[ 9] = 0; - ctx->W64[10] = 0; - ctx->W64[11] = 0; - ctx->W64[12] = 0; - ctx->W64[13] = 0; - ctx->W64[14] = 0; - ctx->W64[15] = swap64 ((u64) (final_len * 8)); - ex = ctx->W64[7] >> 56; - break; - case BLSZ512: make_w_with_offset (ctx, 64, offset, sc, pwbl_len, iv, ks, s_te0, s_te1, s_te2, s_te3, s_te4); - ctx->W64[ 8] = 0x80; - ctx->W64[ 9] = 0; - ctx->W64[10] = 0; - ctx->W64[11] = 0; - ctx->W64[12] = 0; - ctx->W64[13] = 0; - ctx->W64[14] = 0; - ctx->W64[15] = swap64 ((u64) (final_len * 8)); - ex = ctx->W64[7] >> 56; - break; - } - } - else - { - switch (ctx->dgst_len) - { - case BLSZ256: ex = ctx->W32[15] >> 24; - ctx->W32[ 0] = 0x80; - ctx->W32[ 1] = 0; - ctx->W32[ 2] = 0; - ctx->W32[ 3] = 0; - ctx->W32[ 4] = 0; - ctx->W32[ 5] = 0; - ctx->W32[ 6] = 0; - ctx->W32[ 7] = 0; - ctx->W32[ 8] = 0; - ctx->W32[ 9] = 0; - ctx->W32[10] = 0; - ctx->W32[11] = 0; - ctx->W32[12] = 0; - ctx->W32[13] = 0; - ctx->W32[14] = 0; - ctx->W32[15] = swap32 (final_len * 8); - break; - case BLSZ384: ex = ctx->W64[15] >> 56; - ctx->W64[ 0] = 0x80; - ctx->W64[ 1] = 0; - ctx->W64[ 2] = 0; - ctx->W64[ 3] = 0; - ctx->W64[ 4] = 0; - ctx->W64[ 5] = 0; - ctx->W64[ 6] = 0; - ctx->W64[ 7] = 0; - ctx->W64[ 8] = 0; - ctx->W64[ 9] = 0; - ctx->W64[10] = 0; - ctx->W64[11] = 0; - ctx->W64[12] = 0; - ctx->W64[13] = 0; - ctx->W64[14] = 0; - ctx->W64[15] = swap64 ((u64) (final_len * 8)); - break; - case BLSZ512: ex = ctx->W64[15] >> 56; - ctx->W64[ 0] = 0x80; - ctx->W64[ 1] = 0; - ctx->W64[ 2] = 0; - ctx->W64[ 3] = 0; - ctx->W64[ 4] = 0; - ctx->W64[ 5] = 0; - ctx->W64[ 6] = 0; - ctx->W64[ 7] = 0; - ctx->W64[ 8] = 0; - ctx->W64[ 9] = 0; - ctx->W64[10] = 0; - ctx->W64[11] = 0; - ctx->W64[12] = 0; - ctx->W64[13] = 0; - ctx->W64[14] = 0; - ctx->W64[15] = swap64 ((u64) (final_len * 8)); - break; - } - } + sha256_ctx_t ctx256; - switch (ctx->dgst_len) + sha256_init (&ctx256); + + for (int i = 0; i < 64; i++) + { + sha256_update_aes (&ctx256, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + w0[0] = tmp->dgst32[ 0]; + w0[1] = tmp->dgst32[ 1]; + w0[2] = tmp->dgst32[ 2]; + w0[3] = tmp->dgst32[ 3]; + w1[0] = tmp->dgst32[ 4]; + w1[1] = tmp->dgst32[ 5]; + w1[2] = tmp->dgst32[ 6]; + w1[3] = tmp->dgst32[ 7]; + w2[0] = tmp->dgst32[ 8]; + w2[1] = tmp->dgst32[ 9]; + w2[2] = tmp->dgst32[10]; + w2[3] = tmp->dgst32[11]; + w3[0] = tmp->dgst32[12]; + w3[1] = tmp->dgst32[13]; + w3[2] = tmp->dgst32[14]; + w3[3] = tmp->dgst32[15]; + w4[0] = 0; + w4[1] = 0; + w4[2] = 0; + w4[3] = 0; + w5[0] = 0; + w5[1] = 0; + w5[2] = 0; + w5[3] = 0; + w6[0] = 0; + w6[1] = 0; + w6[2] = 0; + w6[3] = 0; + w7[0] = 0; + w7[1] = 0; + w7[2] = 0; + w7[3] = 0; + + ex = sha256_update_aes_64 (&ctx256, w0, w1, w2, w3, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } + + sha256_final_aes (&ctx256, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + tmp->dgst32[ 0] = ctx256.h[0]; + tmp->dgst32[ 1] = ctx256.h[1]; + tmp->dgst32[ 2] = ctx256.h[2]; + tmp->dgst32[ 3] = ctx256.h[3]; + tmp->dgst32[ 4] = ctx256.h[4]; + tmp->dgst32[ 5] = ctx256.h[5]; + tmp->dgst32[ 6] = ctx256.h[6]; + tmp->dgst32[ 7] = ctx256.h[7]; + tmp->dgst32[ 8] = 0; + tmp->dgst32[ 9] = 0; + tmp->dgst32[10] = 0; + tmp->dgst32[11] = 0; + tmp->dgst32[12] = 0; + tmp->dgst32[13] = 0; + tmp->dgst32[14] = 0; + tmp->dgst32[15] = 0; + + tmp->dgst_len = 32; + } + else if (sum3 == 1) { - case BLSZ256: orig_sha256_transform (&ctx->W32[ 0], &ctx->W32[ 4], &ctx->W32[ 8], &ctx->W32[12], ctx->dgst32); - ctx->dgst32[ 0] = swap32 (ctx->dgst32[0]); - ctx->dgst32[ 1] = swap32 (ctx->dgst32[1]); - ctx->dgst32[ 2] = swap32 (ctx->dgst32[2]); - ctx->dgst32[ 3] = swap32 (ctx->dgst32[3]); - ctx->dgst32[ 4] = swap32 (ctx->dgst32[4]); - ctx->dgst32[ 5] = swap32 (ctx->dgst32[5]); - ctx->dgst32[ 6] = swap32 (ctx->dgst32[6]); - ctx->dgst32[ 7] = swap32 (ctx->dgst32[7]); - ctx->dgst32[ 8] = 0; - ctx->dgst32[ 9] = 0; - ctx->dgst32[10] = 0; - ctx->dgst32[11] = 0; - ctx->dgst32[12] = 0; - ctx->dgst32[13] = 0; - ctx->dgst32[14] = 0; - ctx->dgst32[15] = 0; - break; - case BLSZ384: orig_sha384_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - ctx->dgst64[0] = swap64 (ctx->dgst64[0]); - ctx->dgst64[1] = swap64 (ctx->dgst64[1]); - ctx->dgst64[2] = swap64 (ctx->dgst64[2]); - ctx->dgst64[3] = swap64 (ctx->dgst64[3]); - ctx->dgst64[4] = swap64 (ctx->dgst64[4]); - ctx->dgst64[5] = swap64 (ctx->dgst64[5]); - ctx->dgst64[6] = 0; - ctx->dgst64[7] = 0; - break; - case BLSZ512: orig_sha512_transform (&ctx->W64[ 0], &ctx->W64[ 4], &ctx->W64[ 8], &ctx->W64[12], ctx->dgst64); - ctx->dgst64[0] = swap64 (ctx->dgst64[0]); - ctx->dgst64[1] = swap64 (ctx->dgst64[1]); - ctx->dgst64[2] = swap64 (ctx->dgst64[2]); - ctx->dgst64[3] = swap64 (ctx->dgst64[3]); - ctx->dgst64[4] = swap64 (ctx->dgst64[4]); - ctx->dgst64[5] = swap64 (ctx->dgst64[5]); - ctx->dgst64[6] = swap64 (ctx->dgst64[6]); - ctx->dgst64[7] = swap64 (ctx->dgst64[7]); - break; + sha384_ctx_t ctx384; + + sha384_init (&ctx384); + + for (int i = 0; i < 64; i++) + { + sha384_update_aes (&ctx384, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + w0[0] = tmp->dgst32[ 0]; + w0[1] = tmp->dgst32[ 1]; + w0[2] = tmp->dgst32[ 2]; + w0[3] = tmp->dgst32[ 3]; + w1[0] = tmp->dgst32[ 4]; + w1[1] = tmp->dgst32[ 5]; + w1[2] = tmp->dgst32[ 6]; + w1[3] = tmp->dgst32[ 7]; + w2[0] = tmp->dgst32[ 8]; + w2[1] = tmp->dgst32[ 9]; + w2[2] = tmp->dgst32[10]; + w2[3] = tmp->dgst32[11]; + w3[0] = tmp->dgst32[12]; + w3[1] = tmp->dgst32[13]; + w3[2] = tmp->dgst32[14]; + w3[3] = tmp->dgst32[15]; + w4[0] = 0; + w4[1] = 0; + w4[2] = 0; + w4[3] = 0; + w5[0] = 0; + w5[1] = 0; + w5[2] = 0; + w5[3] = 0; + w6[0] = 0; + w6[1] = 0; + w6[2] = 0; + w6[3] = 0; + w7[0] = 0; + w7[1] = 0; + w7[2] = 0; + w7[3] = 0; + + sha384_update_aes_128 (&ctx384, w0, w1, w2, w3, w4, w5, w6, w7, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } + + sha384_final_aes (&ctx384, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + ex = ctx384.w3[3] & 0xff; + + tmp->dgst32[ 0] = h32_from_64_S (ctx384.h[0]); + tmp->dgst32[ 1] = l32_from_64_S (ctx384.h[0]); + tmp->dgst32[ 2] = h32_from_64_S (ctx384.h[1]); + tmp->dgst32[ 3] = l32_from_64_S (ctx384.h[1]); + tmp->dgst32[ 4] = h32_from_64_S (ctx384.h[2]); + tmp->dgst32[ 5] = l32_from_64_S (ctx384.h[2]); + tmp->dgst32[ 6] = h32_from_64_S (ctx384.h[3]); + tmp->dgst32[ 7] = l32_from_64_S (ctx384.h[3]); + tmp->dgst32[ 8] = h32_from_64_S (ctx384.h[4]); + tmp->dgst32[ 9] = l32_from_64_S (ctx384.h[4]); + tmp->dgst32[10] = h32_from_64_S (ctx384.h[5]); + tmp->dgst32[11] = l32_from_64_S (ctx384.h[5]); + tmp->dgst32[12] = 0; + tmp->dgst32[13] = 0; + tmp->dgst32[14] = 0; + tmp->dgst32[15] = 0; + + tmp->dgst_len = 48; + } + else if (sum3 == 2) + { + sha512_ctx_t ctx512; + + sha512_init (&ctx512); + + for (int i = 0; i < 64; i++) + { + sha512_update_aes (&ctx512, w, pw_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + w0[0] = tmp->dgst32[ 0]; + w0[1] = tmp->dgst32[ 1]; + w0[2] = tmp->dgst32[ 2]; + w0[3] = tmp->dgst32[ 3]; + w1[0] = tmp->dgst32[ 4]; + w1[1] = tmp->dgst32[ 5]; + w1[2] = tmp->dgst32[ 6]; + w1[3] = tmp->dgst32[ 7]; + w2[0] = tmp->dgst32[ 8]; + w2[1] = tmp->dgst32[ 9]; + w2[2] = tmp->dgst32[10]; + w2[3] = tmp->dgst32[11]; + w3[0] = tmp->dgst32[12]; + w3[1] = tmp->dgst32[13]; + w3[2] = tmp->dgst32[14]; + w3[3] = tmp->dgst32[15]; + w4[0] = 0; + w4[1] = 0; + w4[2] = 0; + w4[3] = 0; + w5[0] = 0; + w5[1] = 0; + w5[2] = 0; + w5[3] = 0; + w6[0] = 0; + w6[1] = 0; + w6[2] = 0; + w6[3] = 0; + w7[0] = 0; + w7[1] = 0; + w7[2] = 0; + w7[3] = 0; + + sha512_update_aes_128 (&ctx512, w0, w1, w2, w3, w4, w5, w6, w7, tmp->dgst_len, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + } + + sha512_final_aes (&ctx512, aes_ks, aes_iv, s_te0, s_te1, s_te2, s_te3, s_te4); + + ex = ctx512.w3[3] & 0xff; + + tmp->dgst32[ 0] = h32_from_64_S (ctx512.h[0]); + tmp->dgst32[ 1] = l32_from_64_S (ctx512.h[0]); + tmp->dgst32[ 2] = h32_from_64_S (ctx512.h[1]); + tmp->dgst32[ 3] = l32_from_64_S (ctx512.h[1]); + tmp->dgst32[ 4] = h32_from_64_S (ctx512.h[2]); + tmp->dgst32[ 5] = l32_from_64_S (ctx512.h[2]); + tmp->dgst32[ 6] = h32_from_64_S (ctx512.h[3]); + tmp->dgst32[ 7] = l32_from_64_S (ctx512.h[3]); + tmp->dgst32[ 8] = h32_from_64_S (ctx512.h[4]); + tmp->dgst32[ 9] = l32_from_64_S (ctx512.h[4]); + tmp->dgst32[10] = h32_from_64_S (ctx512.h[5]); + tmp->dgst32[11] = l32_from_64_S (ctx512.h[5]); + tmp->dgst32[12] = h32_from_64_S (ctx512.h[6]); + tmp->dgst32[13] = l32_from_64_S (ctx512.h[6]); + tmp->dgst32[14] = h32_from_64_S (ctx512.h[7]); + tmp->dgst32[15] = l32_from_64_S (ctx512.h[7]); + + tmp->dgst_len = 64; } return ex; @@ -687,16 +1124,28 @@ __kernel void m10700_init (__global pw_t *pws, __global const kernel_rule_t *rul sha256_final (&ctx); - tmps[gid].dgst32[0] = swap32_S (ctx.h[0]); - tmps[gid].dgst32[1] = swap32_S (ctx.h[1]); - tmps[gid].dgst32[2] = swap32_S (ctx.h[2]); - tmps[gid].dgst32[3] = swap32_S (ctx.h[3]); - tmps[gid].dgst32[4] = swap32_S (ctx.h[4]); - tmps[gid].dgst32[5] = swap32_S (ctx.h[5]); - tmps[gid].dgst32[6] = swap32_S (ctx.h[6]); - tmps[gid].dgst32[7] = swap32_S (ctx.h[7]); - tmps[gid].dgst_len = BLSZ256; - tmps[gid].W_len = WORDSZ256; + pdf17l8_tmp_t tmp; + + tmp.dgst32[ 0] = ctx.h[0]; + tmp.dgst32[ 1] = ctx.h[1]; + tmp.dgst32[ 2] = ctx.h[2]; + tmp.dgst32[ 3] = ctx.h[3]; + tmp.dgst32[ 4] = ctx.h[4]; + tmp.dgst32[ 5] = ctx.h[5]; + tmp.dgst32[ 6] = ctx.h[6]; + tmp.dgst32[ 7] = ctx.h[7]; + tmp.dgst32[ 8] = 0; + tmp.dgst32[ 9] = 0; + tmp.dgst32[10] = 0; + tmp.dgst32[11] = 0; + tmp.dgst32[12] = 0; + tmp.dgst32[13] = 0; + tmp.dgst32[14] = 0; + tmp.dgst32[15] = 0; + + tmp.dgst_len = 32; + + tmps[gid] = tmp; } __kernel void m10700_loop (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -744,59 +1193,41 @@ __kernel void m10700_loop (__global pw_t *pws, __global const kernel_rule_t *rul * base */ - u32 w0[4]; - - w0[0] = pws[gid].i[0]; - w0[1] = pws[gid].i[1]; - w0[2] = pws[gid].i[2]; - w0[3] = pws[gid].i[3]; - const u32 pw_len = pws[gid].pw_len; - if (pw_len == 0) return; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * digest */ - ctx_t ctx; - - ctx.dgst64[0] = tmps[gid].dgst64[0]; - ctx.dgst64[1] = tmps[gid].dgst64[1]; - ctx.dgst64[2] = tmps[gid].dgst64[2]; - ctx.dgst64[3] = tmps[gid].dgst64[3]; - ctx.dgst64[4] = tmps[gid].dgst64[4]; - ctx.dgst64[5] = tmps[gid].dgst64[5]; - ctx.dgst64[6] = tmps[gid].dgst64[6]; - ctx.dgst64[7] = tmps[gid].dgst64[7]; - ctx.dgst_len = tmps[gid].dgst_len; - ctx.W_len = tmps[gid].W_len; + pdf17l8_tmp_t tmp = tmps[gid]; u32 ex = 0; for (u32 i = 0, j = loop_pos; i < loop_cnt; i++, j++) { - ex = do_round (w0, pw_len, &ctx, s_te0, s_te1, s_te2, s_te3, s_te4); + ex = do_round (w, pw_len, &tmp, s_te0, s_te1, s_te2, s_te3, s_te4); } if ((loop_pos + loop_cnt) == 64) { for (u32 i = 64; i < (ex & 0xff) + 32; i++) { - ex = do_round (w0, pw_len, &ctx, s_te0, s_te1, s_te2, s_te3, s_te4); + ex = do_round (w, pw_len, &tmp, s_te0, s_te1, s_te2, s_te3, s_te4); } } - tmps[gid].dgst64[0] = ctx.dgst64[0]; - tmps[gid].dgst64[1] = ctx.dgst64[1]; - tmps[gid].dgst64[2] = ctx.dgst64[2]; - tmps[gid].dgst64[3] = ctx.dgst64[3]; - tmps[gid].dgst64[4] = ctx.dgst64[4]; - tmps[gid].dgst64[5] = ctx.dgst64[5]; - tmps[gid].dgst64[6] = ctx.dgst64[6]; - tmps[gid].dgst64[7] = ctx.dgst64[7]; - tmps[gid].dgst_len = ctx.dgst_len; - tmps[gid].W_len = ctx.W_len; + tmps[gid] = tmp; } __kernel void m10700_comp (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global pdf17l8_tmp_t *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const pdf_t *pdf_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) @@ -815,10 +1246,10 @@ __kernel void m10700_comp (__global pw_t *pws, __global const kernel_rule_t *rul * digest */ - const u32 r0 = swap32_S (tmps[gid].dgst32[DGST_R0]); - const u32 r1 = swap32_S (tmps[gid].dgst32[DGST_R1]); - const u32 r2 = swap32_S (tmps[gid].dgst32[DGST_R2]); - const u32 r3 = swap32_S (tmps[gid].dgst32[DGST_R3]); + const u32 r0 = tmps[gid].dgst32[DGST_R0]; + const u32 r1 = tmps[gid].dgst32[DGST_R1]; + const u32 r2 = tmps[gid].dgst32[DGST_R2]; + const u32 r3 = tmps[gid].dgst32[DGST_R3]; #define il_pos 0 diff --git a/OpenCL/m10800_a0-optimized.cl b/OpenCL/m10800_a0-optimized.cl new file mode 100644 index 000000000..6ef2e2d3a --- /dev/null +++ b/OpenCL/m10800_a0-optimized.cl @@ -0,0 +1,352 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_rp.h" +#include "inc_rp.cl" +#include "inc_simd.cl" + +__constant u64a k_sha384[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = 0; + digest[7] = 0; +} + +__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_len * 8; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + u32x w0[4] = { 0 }; + u32x w1[4] = { 0 }; + u32x w2[4] = { 0 }; + u32x w3[4] = { 0 }; + + const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + + append_0x80_2x4_VV (w0, w1, out_len); + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = out_len * 8; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m10800_a0-pure.cl b/OpenCL/m10800_a0-pure.cl deleted file mode 100644 index 1ae43d030..000000000 --- a/OpenCL/m10800_a0-pure.cl +++ /dev/null @@ -1,134 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_rp.h" -#include "inc_rp.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha384.cl" - -__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha384_ctx_t ctx; - - sha384_init (&ctx); - - sha384_update (&ctx, w, pw_len); - - sha384_final (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32 w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = swap32_S (pws[gid].i[idx]); - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - // todo: add rules engine - - sha384_ctx_t ctx; - - sha384_init (&ctx); - - sha384_update (&ctx, w, pw_len); - - sha384_final (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m10800_a0.cl b/OpenCL/m10800_a0.cl index 6ef2e2d3a..1ae43d030 100644 --- a/OpenCL/m10800_a0.cl +++ b/OpenCL/m10800_a0.cl @@ -3,7 +3,7 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" @@ -12,262 +12,73 @@ #include "inc_common.cl" #include "inc_rp.h" #include "inc_rp.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha384.cl" -__constant u64a k_sha384[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = 0; - digest[7] = 0; -} - -__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - const u32 pw_len = pws[gid].pw_len; + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha384_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha384_init (&ctx); - /** - * sha512 - */ + sha384_update (&ctx, w, pw_len); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha384_final (&ctx); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_len * 8; + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); - u64x digest[8]; - - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; - - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_len = pws[gid].pw_len; - /** * digest */ @@ -280,73 +91,44 @@ __kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32 w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = swap32_S (pws[gid].i[idx]); + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - u32x w0[4] = { 0 }; - u32x w1[4] = { 0 }; - u32x w2[4] = { 0 }; - u32x w3[4] = { 0 }; + // todo: add rules engine - const u32x out_len = apply_rules_vect (pw_buf0, pw_buf1, pw_len, rules_buf, il_pos, w0, w1); + sha384_ctx_t ctx; - append_0x80_2x4_VV (w0, w1, out_len); + sha384_init (&ctx); - /** - * sha512 - */ + sha384_update (&ctx, w, pw_len); - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + sha384_final (&ctx); - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = out_len * 8; + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); - u64x digest[8]; - - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; - - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m10800_a1-optimized.cl b/OpenCL/m10800_a1-optimized.cl new file mode 100644 index 000000000..401609672 --- /dev/null +++ b/OpenCL/m10800_a1-optimized.cl @@ -0,0 +1,466 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha384[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = 0; + digest[7] = 0; +} + +__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_len * 8; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * modifier + */ + + const u32 lid = get_local_id (0); + + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 pw_buf0[4]; + u32 pw_buf1[4]; + + pw_buf0[0] = pws[gid].i[0]; + pw_buf0[1] = pws[gid].i[1]; + pw_buf0[2] = pws[gid].i[2]; + pw_buf0[3] = pws[gid].i[3]; + pw_buf1[0] = pws[gid].i[4]; + pw_buf1[1] = pws[gid].i[5]; + pw_buf1[2] = pws[gid].i[6]; + pw_buf1[3] = pws[gid].i[7]; + + const u32 pw_l_len = pws[gid].pw_len; + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + + const u32x pw_len = pw_l_len + pw_r_len; + + /** + * concat password candidate + */ + + u32x wordl0[4] = { 0 }; + u32x wordl1[4] = { 0 }; + u32x wordl2[4] = { 0 }; + u32x wordl3[4] = { 0 }; + + wordl0[0] = pw_buf0[0]; + wordl0[1] = pw_buf0[1]; + wordl0[2] = pw_buf0[2]; + wordl0[3] = pw_buf0[3]; + wordl1[0] = pw_buf1[0]; + wordl1[1] = pw_buf1[1]; + wordl1[2] = pw_buf1[2]; + wordl1[3] = pw_buf1[3]; + + u32x wordr0[4] = { 0 }; + u32x wordr1[4] = { 0 }; + u32x wordr2[4] = { 0 }; + u32x wordr3[4] = { 0 }; + + wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); + wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); + wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); + wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); + wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); + wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); + wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); + wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); + + if (combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); + } + else + { + switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); + } + + u32x w0[4]; + u32x w1[4]; + u32x w2[4]; + u32x w3[4]; + + w0[0] = wordl0[0] | wordr0[0]; + w0[1] = wordl0[1] | wordr0[1]; + w0[2] = wordl0[2] | wordr0[2]; + w0[3] = wordl0[3] | wordr0[3]; + w1[0] = wordl1[0] | wordr1[0]; + w1[1] = wordl1[1] | wordr1[1]; + w1[2] = wordl1[2] | wordr1[2]; + w1[3] = wordl1[3] | wordr1[3]; + w2[0] = wordl2[0] | wordr2[0]; + w2[1] = wordl2[1] | wordr2[1]; + w2[2] = wordl2[2] | wordr2[2]; + w2[3] = wordl2[3] | wordr2[3]; + w3[0] = wordl3[0] | wordr3[0]; + w3[1] = wordl3[1] | wordr3[1]; + w3[2] = wordl3[2] | wordr3[2]; + w3[3] = wordl3[3] | wordr3[3]; + + /** + * sha512 + */ + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = swap32 (w0[0]); + w0_t[1] = swap32 (w0[1]); + w0_t[2] = swap32 (w0[2]); + w0_t[3] = swap32 (w0[3]); + w1_t[0] = swap32 (w1[0]); + w1_t[1] = swap32 (w1[1]); + w1_t[2] = swap32 (w1[2]); + w1_t[3] = swap32 (w1[3]); + w2_t[0] = swap32 (w2[0]); + w2_t[1] = swap32 (w2[1]); + w2_t[2] = swap32 (w2[2]); + w2_t[3] = swap32 (w2[3]); + w3_t[0] = swap32 (w3[0]); + w3_t[1] = swap32 (w3[1]); + w3_t[2] = 0; + w3_t[3] = pw_len * 8; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} + +__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ +} diff --git a/OpenCL/m10800_a1-pure.cl b/OpenCL/m10800_a1-pure.cl deleted file mode 100644 index 9f1eded3a..000000000 --- a/OpenCL/m10800_a1-pure.cl +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -//#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_scalar.cl" -#include "inc_hash_sha384.cl" - -__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - sha384_ctx_t ctx0; - - sha384_init (&ctx0); - - sha384_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha384_ctx_t ctx = ctx0; - - sha384_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha384_final (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_M_SCALAR (r0, r1, r2, r3); - } -} - -__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - sha384_ctx_t ctx0; - - sha384_init (&ctx0); - - sha384_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); - - /** - * loop - */ - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) - { - sha384_ctx_t ctx = ctx0; - - sha384_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - - sha384_final (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_S_SCALAR (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m10800_a1.cl b/OpenCL/m10800_a1.cl index 401609672..9f1eded3a 100644 --- a/OpenCL/m10800_a1.cl +++ b/OpenCL/m10800_a1.cl @@ -3,327 +3,69 @@ * License.....: MIT */ -#define NEW_SIMD_CODE +//#define NEW_SIMD_CODE #include "inc_vendor.cl" #include "inc_hash_constants.h" #include "inc_hash_functions.cl" #include "inc_types.cl" #include "inc_common.cl" -#include "inc_simd.cl" +#include "inc_scalar.cl" +#include "inc_hash_sha384.cl" -__constant u64a k_sha384[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = 0; - digest[7] = 0; -} - -__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * base */ - const u32 gid = get_global_id (0); + sha384_ctx_t ctx0; - if (gid >= gid_max) return; + sha384_init (&ctx0); - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; + sha384_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha384_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha384_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha384_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_len * 8; - - u64x digest[8]; - - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; - - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); - - COMPARE_M_SIMD (r0, r1, r2, r3); + COMPARE_M_SCALAR (r0, r1, r2, r3); } } -__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ const u32 lid = get_local_id (0); - - /** - * base - */ - const u32 gid = get_global_id (0); if (gid >= gid_max) return; - u32 pw_buf0[4]; - u32 pw_buf1[4]; - - pw_buf0[0] = pws[gid].i[0]; - pw_buf0[1] = pws[gid].i[1]; - pw_buf0[2] = pws[gid].i[2]; - pw_buf0[3] = pws[gid].i[3]; - pw_buf1[0] = pws[gid].i[4]; - pw_buf1[1] = pws[gid].i[5]; - pw_buf1[2] = pws[gid].i[6]; - pw_buf1[3] = pws[gid].i[7]; - - const u32 pw_l_len = pws[gid].pw_len; - /** * digest */ @@ -336,131 +78,33 @@ __kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rule digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + sha384_ctx_t ctx0; + + sha384_init (&ctx0); + + sha384_update_global_swap (&ctx0, pws[gid].i, pws[gid].pw_len); + /** * loop */ - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + for (u32 il_pos = 0; il_pos < il_cnt; il_pos++) { - const u32x pw_r_len = pwlenx_create_combt (combs_buf, il_pos); + sha384_ctx_t ctx = ctx0; - const u32x pw_len = pw_l_len + pw_r_len; + sha384_update_global_swap (&ctx, combs_buf[il_pos].i, combs_buf[il_pos].pw_len); - /** - * concat password candidate - */ + sha384_final (&ctx); - u32x wordl0[4] = { 0 }; - u32x wordl1[4] = { 0 }; - u32x wordl2[4] = { 0 }; - u32x wordl3[4] = { 0 }; + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); - wordl0[0] = pw_buf0[0]; - wordl0[1] = pw_buf0[1]; - wordl0[2] = pw_buf0[2]; - wordl0[3] = pw_buf0[3]; - wordl1[0] = pw_buf1[0]; - wordl1[1] = pw_buf1[1]; - wordl1[2] = pw_buf1[2]; - wordl1[3] = pw_buf1[3]; - - u32x wordr0[4] = { 0 }; - u32x wordr1[4] = { 0 }; - u32x wordr2[4] = { 0 }; - u32x wordr3[4] = { 0 }; - - wordr0[0] = ix_create_combt (combs_buf, il_pos, 0); - wordr0[1] = ix_create_combt (combs_buf, il_pos, 1); - wordr0[2] = ix_create_combt (combs_buf, il_pos, 2); - wordr0[3] = ix_create_combt (combs_buf, il_pos, 3); - wordr1[0] = ix_create_combt (combs_buf, il_pos, 4); - wordr1[1] = ix_create_combt (combs_buf, il_pos, 5); - wordr1[2] = ix_create_combt (combs_buf, il_pos, 6); - wordr1[3] = ix_create_combt (combs_buf, il_pos, 7); - - if (combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - switch_buffer_by_offset_le_VV (wordr0, wordr1, wordr2, wordr3, pw_l_len); - } - else - { - switch_buffer_by_offset_le_VV (wordl0, wordl1, wordl2, wordl3, pw_r_len); - } - - u32x w0[4]; - u32x w1[4]; - u32x w2[4]; - u32x w3[4]; - - w0[0] = wordl0[0] | wordr0[0]; - w0[1] = wordl0[1] | wordr0[1]; - w0[2] = wordl0[2] | wordr0[2]; - w0[3] = wordl0[3] | wordr0[3]; - w1[0] = wordl1[0] | wordr1[0]; - w1[1] = wordl1[1] | wordr1[1]; - w1[2] = wordl1[2] | wordr1[2]; - w1[3] = wordl1[3] | wordr1[3]; - w2[0] = wordl2[0] | wordr2[0]; - w2[1] = wordl2[1] | wordr2[1]; - w2[2] = wordl2[2] | wordr2[2]; - w2[3] = wordl2[3] | wordr2[3]; - w3[0] = wordl3[0] | wordr3[0]; - w3[1] = wordl3[1] | wordr3[1]; - w3[2] = wordl3[2] | wordr3[2]; - w3[3] = wordl3[3] | wordr3[3]; - - /** - * sha512 - */ - - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; - - w0_t[0] = swap32 (w0[0]); - w0_t[1] = swap32 (w0[1]); - w0_t[2] = swap32 (w0[2]); - w0_t[3] = swap32 (w0[3]); - w1_t[0] = swap32 (w1[0]); - w1_t[1] = swap32 (w1[1]); - w1_t[2] = swap32 (w1[2]); - w1_t[3] = swap32 (w1[3]); - w2_t[0] = swap32 (w2[0]); - w2_t[1] = swap32 (w2[1]); - w2_t[2] = swap32 (w2[2]); - w2_t[3] = swap32 (w2[3]); - w3_t[0] = swap32 (w3[0]); - w3_t[1] = swap32 (w3[1]); - w3_t[2] = 0; - w3_t[3] = pw_len * 8; - - u64x digest[8]; - - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; - - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); - - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); - - COMPARE_S_SIMD (r0, r1, r2, r3); + COMPARE_S_SCALAR (r0, r1, r2, r3); } } - -__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} - -__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __global const bf_t *bfs_buf, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ -} diff --git a/OpenCL/m10800_a3-optimized.cl b/OpenCL/m10800_a3-optimized.cl new file mode 100644 index 000000000..141c5617b --- /dev/null +++ b/OpenCL/m10800_a3-optimized.cl @@ -0,0 +1,506 @@ +/** + * Author......: See docs/credits.txt + * License.....: MIT + */ + +#define NEW_SIMD_CODE + +#include "inc_vendor.cl" +#include "inc_hash_constants.h" +#include "inc_hash_functions.cl" +#include "inc_types.cl" +#include "inc_common.cl" +#include "inc_simd.cl" + +__constant u64a k_sha384[80] = +{ + SHA512C00, SHA512C01, SHA512C02, SHA512C03, + SHA512C04, SHA512C05, SHA512C06, SHA512C07, + SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, + SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, + SHA512C10, SHA512C11, SHA512C12, SHA512C13, + SHA512C14, SHA512C15, SHA512C16, SHA512C17, + SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, + SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, + SHA512C20, SHA512C21, SHA512C22, SHA512C23, + SHA512C24, SHA512C25, SHA512C26, SHA512C27, + SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, + SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, + SHA512C30, SHA512C31, SHA512C32, SHA512C33, + SHA512C34, SHA512C35, SHA512C36, SHA512C37, + SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, + SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, + SHA512C40, SHA512C41, SHA512C42, SHA512C43, + SHA512C44, SHA512C45, SHA512C46, SHA512C47, + SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, + SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, +}; + +void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) +{ + u64x w0_t = hl32_to_64 (w0[0], w0[1]); + u64x w1_t = hl32_to_64 (w0[2], w0[3]); + u64x w2_t = hl32_to_64 (w1[0], w1[1]); + u64x w3_t = hl32_to_64 (w1[2], w1[3]); + u64x w4_t = hl32_to_64 (w2[0], w2[1]); + u64x w5_t = hl32_to_64 (w2[2], w2[3]); + u64x w6_t = hl32_to_64 (w3[0], w3[1]); + u64x w7_t = 0; + u64x w8_t = 0; + u64x w9_t = 0; + u64x wa_t = 0; + u64x wb_t = 0; + u64x wc_t = 0; + u64x wd_t = 0; + u64x we_t = 0; + u64x wf_t = hl32_to_64 (w3[2], w3[3]); + + u64x a = digest[0]; + u64x b = digest[1]; + u64x c = digest[2]; + u64x d = digest[3]; + u64x e = digest[4]; + u64x f = digest[5]; + u64x g = digest[6]; + u64x h = digest[7]; + + #define ROUND_EXPAND() \ + { \ + w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ + w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ + w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ + w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ + w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ + w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ + w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ + w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ + w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ + w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ + wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ + wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ + wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ + wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ + we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ + wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ + } + + #define ROUND_STEP(i) \ + { \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ + SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ + } + + ROUND_STEP (0); + + #ifdef _unroll + #pragma unroll + #endif + for (int i = 16; i < 80; i += 16) + { + ROUND_EXPAND (); ROUND_STEP (i); + } + + /* rev + digest[0] += a; + digest[1] += b; + digest[2] += c; + digest[3] += d; + digest[4] += e; + digest[5] += f; + digest[6] += g; + digest[7] += h; + */ + + digest[0] = a; + digest[1] = b; + digest[2] = c; + digest[3] = d; + digest[4] = e; + digest[5] = f; + digest[6] = 0; + digest[7] = 0; +} + +void m10800m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_M_SIMD (r0, r1, r2, r3); + } +} + +void m10800s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +{ + /** + * modifier + */ + + const u32 gid = get_global_id (0); + const u32 lid = get_local_id (0); + + /** + * digest + */ + + const u32 search[4] = + { + digests_buf[digests_offset].digest_buf[DGST_R0], + digests_buf[digests_offset].digest_buf[DGST_R1], + digests_buf[digests_offset].digest_buf[DGST_R2], + digests_buf[digests_offset].digest_buf[DGST_R3] + }; + + /** + * loop + */ + + u32 w0l = w[0]; + + for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) + { + const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; + + const u32x w0 = w0l | w0r; + + u32x w0_t[4]; + u32x w1_t[4]; + u32x w2_t[4]; + u32x w3_t[4]; + + w0_t[0] = w0; + w0_t[1] = w[ 1]; + w0_t[2] = w[ 2]; + w0_t[3] = w[ 3]; + w1_t[0] = w[ 4]; + w1_t[1] = w[ 5]; + w1_t[2] = w[ 6]; + w1_t[3] = w[ 7]; + w2_t[0] = w[ 8]; + w2_t[1] = w[ 9]; + w2_t[2] = w[10]; + w2_t[3] = w[11]; + w3_t[0] = w[12]; + w3_t[1] = w[13]; + w3_t[2] = w[14]; + w3_t[3] = w[15]; + + u64x digest[8]; + + digest[0] = SHA384M_A; + digest[1] = SHA384M_B; + digest[2] = SHA384M_C; + digest[3] = SHA384M_D; + digest[4] = SHA384M_E; + digest[5] = SHA384M_F; + digest[6] = SHA384M_G; + digest[7] = SHA384M_H; + + sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + + const u32x r0 = l32_from_64 (digest[3]); + const u32x r1 = h32_from_64 (digest[3]); + const u32x r2 = l32_from_64 (digest[2]); + const u32x r3 = h32_from_64 (digest[2]); + + COMPARE_S_SIMD (r0, r1, r2, r3); + } +} + +__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = 0; + w[ 5] = 0; + w[ 6] = 0; + w[ 7] = 0; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = 0; + w[ 9] = 0; + w[10] = 0; + w[11] = 0; + w[12] = 0; + w[13] = 0; + w[14] = 0; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} + +__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) +{ + /** + * base + */ + + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + u32 w[16]; + + w[ 0] = pws[gid].i[ 0]; + w[ 1] = pws[gid].i[ 1]; + w[ 2] = pws[gid].i[ 2]; + w[ 3] = pws[gid].i[ 3]; + w[ 4] = pws[gid].i[ 4]; + w[ 5] = pws[gid].i[ 5]; + w[ 6] = pws[gid].i[ 6]; + w[ 7] = pws[gid].i[ 7]; + w[ 8] = pws[gid].i[ 8]; + w[ 9] = pws[gid].i[ 9]; + w[10] = pws[gid].i[10]; + w[11] = pws[gid].i[11]; + w[12] = pws[gid].i[12]; + w[13] = pws[gid].i[13]; + w[14] = pws[gid].i[14]; + w[15] = pws[gid].i[15]; + + const u32 pw_len = pws[gid].pw_len; + + /** + * main + */ + + m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); +} diff --git a/OpenCL/m10800_a3-pure.cl b/OpenCL/m10800_a3-pure.cl deleted file mode 100644 index 72ba67211..000000000 --- a/OpenCL/m10800_a3-pure.cl +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Author......: See docs/credits.txt - * License.....: MIT - */ - -#define NEW_SIMD_CODE - -#include "inc_vendor.cl" -#include "inc_hash_constants.h" -#include "inc_hash_functions.cl" -#include "inc_types.cl" -#include "inc_common.cl" -#include "inc_simd.cl" -#include "inc_hash_sha384.cl" - -__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha384_ctx_vector_t ctx; - - sha384_init_vector (&ctx); - - sha384_update_vector (&ctx, w, pw_len); - - sha384_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_M_SIMD (r0, r1, r2, r3); - } -} - -__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * modifier - */ - - const u32 lid = get_local_id (0); - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - /** - * digest - */ - - const u32 search[4] = - { - digests_buf[digests_offset].digest_buf[DGST_R0], - digests_buf[digests_offset].digest_buf[DGST_R1], - digests_buf[digests_offset].digest_buf[DGST_R2], - digests_buf[digests_offset].digest_buf[DGST_R3] - }; - - /** - * base - */ - - const u32 pw_len = pws[gid].pw_len; - - const u32 pw_lenv = ceil ((float) pw_len / 4); - - u32x w[64] = { 0 }; - - for (int idx = 0; idx < pw_lenv; idx++) - { - w[idx] = pws[gid].i[idx]; - - barrier (CLK_GLOBAL_MEM_FENCE); - } - - /** - * loop - */ - - u32x w0l = w[0]; - - for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) - { - const u32x w0r = words_buf_r[il_pos / VECT_SIZE]; - - const u32x w0 = w0l | w0r; - - w[0] = w0; - - sha384_ctx_vector_t ctx; - - sha384_init_vector (&ctx); - - sha384_update_vector (&ctx, w, pw_len); - - sha384_final_vector (&ctx); - - const u32x r0 = l32_from_64 (ctx.h[3]); - const u32x r1 = h32_from_64 (ctx.h[3]); - const u32x r2 = l32_from_64 (ctx.h[2]); - const u32x r3 = h32_from_64 (ctx.h[2]); - - COMPARE_S_SIMD (r0, r1, r2, r3); - } -} diff --git a/OpenCL/m10800_a3.cl b/OpenCL/m10800_a3.cl index 141c5617b..72ba67211 100644 --- a/OpenCL/m10800_a3.cl +++ b/OpenCL/m10800_a3.cl @@ -11,144 +11,41 @@ #include "inc_types.cl" #include "inc_common.cl" #include "inc_simd.cl" +#include "inc_hash_sha384.cl" -__constant u64a k_sha384[80] = -{ - SHA512C00, SHA512C01, SHA512C02, SHA512C03, - SHA512C04, SHA512C05, SHA512C06, SHA512C07, - SHA512C08, SHA512C09, SHA512C0a, SHA512C0b, - SHA512C0c, SHA512C0d, SHA512C0e, SHA512C0f, - SHA512C10, SHA512C11, SHA512C12, SHA512C13, - SHA512C14, SHA512C15, SHA512C16, SHA512C17, - SHA512C18, SHA512C19, SHA512C1a, SHA512C1b, - SHA512C1c, SHA512C1d, SHA512C1e, SHA512C1f, - SHA512C20, SHA512C21, SHA512C22, SHA512C23, - SHA512C24, SHA512C25, SHA512C26, SHA512C27, - SHA512C28, SHA512C29, SHA512C2a, SHA512C2b, - SHA512C2c, SHA512C2d, SHA512C2e, SHA512C2f, - SHA512C30, SHA512C31, SHA512C32, SHA512C33, - SHA512C34, SHA512C35, SHA512C36, SHA512C37, - SHA512C38, SHA512C39, SHA512C3a, SHA512C3b, - SHA512C3c, SHA512C3d, SHA512C3e, SHA512C3f, - SHA512C40, SHA512C41, SHA512C42, SHA512C43, - SHA512C44, SHA512C45, SHA512C46, SHA512C47, - SHA512C48, SHA512C49, SHA512C4a, SHA512C4b, - SHA512C4c, SHA512C4d, SHA512C4e, SHA512C4f, -}; - -void sha384_transform (const u32x w0[4], const u32x w1[4], const u32x w2[4], const u32x w3[4], u64x digest[8]) -{ - u64x w0_t = hl32_to_64 (w0[0], w0[1]); - u64x w1_t = hl32_to_64 (w0[2], w0[3]); - u64x w2_t = hl32_to_64 (w1[0], w1[1]); - u64x w3_t = hl32_to_64 (w1[2], w1[3]); - u64x w4_t = hl32_to_64 (w2[0], w2[1]); - u64x w5_t = hl32_to_64 (w2[2], w2[3]); - u64x w6_t = hl32_to_64 (w3[0], w3[1]); - u64x w7_t = 0; - u64x w8_t = 0; - u64x w9_t = 0; - u64x wa_t = 0; - u64x wb_t = 0; - u64x wc_t = 0; - u64x wd_t = 0; - u64x we_t = 0; - u64x wf_t = hl32_to_64 (w3[2], w3[3]); - - u64x a = digest[0]; - u64x b = digest[1]; - u64x c = digest[2]; - u64x d = digest[3]; - u64x e = digest[4]; - u64x f = digest[5]; - u64x g = digest[6]; - u64x h = digest[7]; - - #define ROUND_EXPAND() \ - { \ - w0_t = SHA384_EXPAND (we_t, w9_t, w1_t, w0_t); \ - w1_t = SHA384_EXPAND (wf_t, wa_t, w2_t, w1_t); \ - w2_t = SHA384_EXPAND (w0_t, wb_t, w3_t, w2_t); \ - w3_t = SHA384_EXPAND (w1_t, wc_t, w4_t, w3_t); \ - w4_t = SHA384_EXPAND (w2_t, wd_t, w5_t, w4_t); \ - w5_t = SHA384_EXPAND (w3_t, we_t, w6_t, w5_t); \ - w6_t = SHA384_EXPAND (w4_t, wf_t, w7_t, w6_t); \ - w7_t = SHA384_EXPAND (w5_t, w0_t, w8_t, w7_t); \ - w8_t = SHA384_EXPAND (w6_t, w1_t, w9_t, w8_t); \ - w9_t = SHA384_EXPAND (w7_t, w2_t, wa_t, w9_t); \ - wa_t = SHA384_EXPAND (w8_t, w3_t, wb_t, wa_t); \ - wb_t = SHA384_EXPAND (w9_t, w4_t, wc_t, wb_t); \ - wc_t = SHA384_EXPAND (wa_t, w5_t, wd_t, wc_t); \ - wd_t = SHA384_EXPAND (wb_t, w6_t, we_t, wd_t); \ - we_t = SHA384_EXPAND (wc_t, w7_t, wf_t, we_t); \ - wf_t = SHA384_EXPAND (wd_t, w8_t, w0_t, wf_t); \ - } - - #define ROUND_STEP(i) \ - { \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w0_t, k_sha384[i + 0]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w1_t, k_sha384[i + 1]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, w2_t, k_sha384[i + 2]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, w3_t, k_sha384[i + 3]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, w4_t, k_sha384[i + 4]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, w5_t, k_sha384[i + 5]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, w6_t, k_sha384[i + 6]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, w7_t, k_sha384[i + 7]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, a, b, c, d, e, f, g, h, w8_t, k_sha384[i + 8]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, h, a, b, c, d, e, f, g, w9_t, k_sha384[i + 9]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, g, h, a, b, c, d, e, f, wa_t, k_sha384[i + 10]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, f, g, h, a, b, c, d, e, wb_t, k_sha384[i + 11]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, e, f, g, h, a, b, c, d, wc_t, k_sha384[i + 12]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, d, e, f, g, h, a, b, c, wd_t, k_sha384[i + 13]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, c, d, e, f, g, h, a, b, we_t, k_sha384[i + 14]); \ - SHA384_STEP (SHA384_F0o, SHA384_F1o, b, c, d, e, f, g, h, a, wf_t, k_sha384[i + 15]); \ - } - - ROUND_STEP (0); - - #ifdef _unroll - #pragma unroll - #endif - for (int i = 16; i < 80; i += 16) - { - ROUND_EXPAND (); ROUND_STEP (i); - } - - /* rev - digest[0] += a; - digest[1] += b; - digest[2] += c; - digest[3] += d; - digest[4] += e; - digest[5] += f; - digest[6] += g; - digest[7] += h; - */ - - digest[0] = a; - digest[1] = b; - digest[2] = c; - digest[3] = d; - digest[4] = e; - digest[5] = f; - digest[6] = 0; - digest[7] = 0; -} - -void m10800m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m10800_mxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; + + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -156,58 +53,35 @@ void m10800m (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha384_ctx_vector_t ctx; - u64x digest[8]; + sha384_init_vector (&ctx); - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; + sha384_update_vector (&ctx, w, pw_len); - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha384_final_vector (&ctx); - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); COMPARE_M_SIMD (r0, r1, r2, r3); } } -void m10800s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset) +__kernel void m10800_sxx (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) { /** * modifier */ - const u32 gid = get_global_id (0); const u32 lid = get_local_id (0); + const u32 gid = get_global_id (0); + + if (gid >= gid_max) return; /** * digest @@ -221,11 +95,28 @@ void m10800s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke digests_buf[digests_offset].digest_buf[DGST_R3] }; + /** + * base + */ + + const u32 pw_len = pws[gid].pw_len; + + const u32 pw_lenv = ceil ((float) pw_len / 4); + + u32x w[64] = { 0 }; + + for (int idx = 0; idx < pw_lenv; idx++) + { + w[idx] = pws[gid].i[idx]; + + barrier (CLK_GLOBAL_MEM_FENCE); + } + /** * loop */ - u32 w0l = w[0]; + u32x w0l = w[0]; for (u32 il_pos = 0; il_pos < il_cnt; il_pos += VECT_SIZE) { @@ -233,274 +124,21 @@ void m10800s (u32 w[16], const u32 pw_len, __global pw_t *pws, __global const ke const u32x w0 = w0l | w0r; - u32x w0_t[4]; - u32x w1_t[4]; - u32x w2_t[4]; - u32x w3_t[4]; + w[0] = w0; - w0_t[0] = w0; - w0_t[1] = w[ 1]; - w0_t[2] = w[ 2]; - w0_t[3] = w[ 3]; - w1_t[0] = w[ 4]; - w1_t[1] = w[ 5]; - w1_t[2] = w[ 6]; - w1_t[3] = w[ 7]; - w2_t[0] = w[ 8]; - w2_t[1] = w[ 9]; - w2_t[2] = w[10]; - w2_t[3] = w[11]; - w3_t[0] = w[12]; - w3_t[1] = w[13]; - w3_t[2] = w[14]; - w3_t[3] = w[15]; + sha384_ctx_vector_t ctx; - u64x digest[8]; + sha384_init_vector (&ctx); - digest[0] = SHA384M_A; - digest[1] = SHA384M_B; - digest[2] = SHA384M_C; - digest[3] = SHA384M_D; - digest[4] = SHA384M_E; - digest[5] = SHA384M_F; - digest[6] = SHA384M_G; - digest[7] = SHA384M_H; + sha384_update_vector (&ctx, w, pw_len); - sha384_transform (w0_t, w1_t, w2_t, w3_t, digest); + sha384_final_vector (&ctx); - const u32x r0 = l32_from_64 (digest[3]); - const u32x r1 = h32_from_64 (digest[3]); - const u32x r2 = l32_from_64 (digest[2]); - const u32x r3 = h32_from_64 (digest[2]); + const u32x r0 = l32_from_64 (ctx.h[3]); + const u32x r1 = h32_from_64 (ctx.h[3]); + const u32x r2 = l32_from_64 (ctx.h[2]); + const u32x r3 = h32_from_64 (ctx.h[2]); COMPARE_S_SIMD (r0, r1, r2, r3); } } - -__kernel void m10800_m04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m10800_m08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m10800_m16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800m (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m10800_s04 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = 0; - w[ 5] = 0; - w[ 6] = 0; - w[ 7] = 0; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m10800_s08 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = 0; - w[ 9] = 0; - w[10] = 0; - w[11] = 0; - w[12] = 0; - w[13] = 0; - w[14] = 0; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} - -__kernel void m10800_s16 (__global pw_t *pws, __global const kernel_rule_t *rules_buf, __global const pw_t *combs_buf, __constant const u32x *words_buf_r, __global void *tmps, __global void *hooks, __global const u32 *bitmaps_buf_s1_a, __global const u32 *bitmaps_buf_s1_b, __global const u32 *bitmaps_buf_s1_c, __global const u32 *bitmaps_buf_s1_d, __global const u32 *bitmaps_buf_s2_a, __global const u32 *bitmaps_buf_s2_b, __global const u32 *bitmaps_buf_s2_c, __global const u32 *bitmaps_buf_s2_d, __global plain_t *plains_buf, __global const digest_t *digests_buf, __global u32 *hashes_shown, __global const salt_t *salt_bufs, __global const void *esalt_bufs, __global u32 *d_return_buf, __global u32 *d_scryptV0_buf, __global u32 *d_scryptV1_buf, __global u32 *d_scryptV2_buf, __global u32 *d_scryptV3_buf, const u32 bitmap_mask, const u32 bitmap_shift1, const u32 bitmap_shift2, const u32 salt_pos, const u32 loop_pos, const u32 loop_cnt, const u32 il_cnt, const u32 digests_cnt, const u32 digests_offset, const u32 combs_mode, const u32 gid_max) -{ - /** - * base - */ - - const u32 gid = get_global_id (0); - - if (gid >= gid_max) return; - - u32 w[16]; - - w[ 0] = pws[gid].i[ 0]; - w[ 1] = pws[gid].i[ 1]; - w[ 2] = pws[gid].i[ 2]; - w[ 3] = pws[gid].i[ 3]; - w[ 4] = pws[gid].i[ 4]; - w[ 5] = pws[gid].i[ 5]; - w[ 6] = pws[gid].i[ 6]; - w[ 7] = pws[gid].i[ 7]; - w[ 8] = pws[gid].i[ 8]; - w[ 9] = pws[gid].i[ 9]; - w[10] = pws[gid].i[10]; - w[11] = pws[gid].i[11]; - w[12] = pws[gid].i[12]; - w[13] = pws[gid].i[13]; - w[14] = pws[gid].i[14]; - w[15] = pws[gid].i[15]; - - const u32 pw_len = pws[gid].pw_len; - - /** - * main - */ - - m10800s (w, pw_len, pws, rules_buf, combs_buf, words_buf_r, tmps, hooks, bitmaps_buf_s1_a, bitmaps_buf_s1_b, bitmaps_buf_s1_c, bitmaps_buf_s1_d, bitmaps_buf_s2_a, bitmaps_buf_s2_b, bitmaps_buf_s2_c, bitmaps_buf_s2_d, plains_buf, digests_buf, hashes_shown, salt_bufs, esalt_bufs, d_return_buf, d_scryptV0_buf, d_scryptV1_buf, d_scryptV2_buf, d_scryptV3_buf, bitmap_mask, bitmap_shift1, bitmap_shift2, salt_pos, loop_pos, loop_cnt, il_cnt, digests_cnt, digests_offset); -} diff --git a/include/types.h b/include/types.h index c65c4a288..d1ee0df63 100644 --- a/include/types.h +++ b/include/types.h @@ -531,7 +531,6 @@ typedef enum user_options_defaults KERNEL_LOOPS = 0, KEYSPACE = false, LEFT = false, - LENGTH_LIMIT_DISABLE = false, LIMIT = 0, LOGFILE_DISABLE = false, LOOPBACK = false, @@ -542,6 +541,7 @@ typedef enum user_options_defaults NONCE_ERROR_CORRECTIONS = 8, NVIDIA_SPIN_DAMP = 100, OPENCL_VECTOR_WIDTH = 0, + OPTIMIZED_KERNEL_ENABLE = false, OUTFILE_AUTOHEX = true, OUTFILE_CHECK_TIMER = 5, OUTFILE_FORMAT = 3, @@ -612,7 +612,6 @@ typedef enum user_options_map IDX_KERNEL_LOOPS = 'u', IDX_KEYSPACE = 0xff14, IDX_LEFT = 0xff15, - IDX_LENGTH_LIMIT_DISABLE = 'L', IDX_LIMIT = 'l', IDX_LOGFILE_DISABLE = 0xff16, IDX_LOOPBACK = 0xff17, @@ -628,6 +627,7 @@ typedef enum user_options_map IDX_OPENCL_INFO = 'I', IDX_OPENCL_PLATFORMS = 0xff1e, IDX_OPENCL_VECTOR_WIDTH = 0xff1f, + IDX_OPTIMIZED_KERNEL_ENABLE = 'O', IDX_OUTFILE_AUTOHEX_DISABLE = 0xff20, IDX_OUTFILE_CHECK_DIR = 0xff21, IDX_OUTFILE_CHECK_TIMER = 0xff22, @@ -1460,13 +1460,13 @@ typedef struct user_options bool keep_guessing; bool keyspace; bool left; - bool length_limit_disable; bool logfile_disable; bool loopback; bool machine_readable; bool markov_classic; bool markov_disable; bool opencl_info; + bool optimized_kernel_enable; bool outfile_autohex; bool potfile_disable; bool powertune_enable; diff --git a/src/autotune.c b/src/autotune.c index 2d324fa10..d997e8b7c 100644 --- a/src/autotune.c +++ b/src/autotune.c @@ -22,18 +22,18 @@ static double try_run (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_par if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { - if (user_options->length_limit_disable == true) - { - const u32 kernel_power_try = device_param->device_processors * device_param->kernel_threads_by_wgs_kernel4 * kernel_accel; - - run_kernel (hashcat_ctx, device_param, KERN_RUN_4, kernel_power_try, true, 0); - } - else + if (user_options->optimized_kernel_enable == true) { const u32 kernel_power_try = device_param->device_processors * device_param->kernel_threads_by_wgs_kernel1 * kernel_accel; run_kernel (hashcat_ctx, device_param, KERN_RUN_1, kernel_power_try, true, 0); } + else + { + const u32 kernel_power_try = device_param->device_processors * device_param->kernel_threads_by_wgs_kernel4 * kernel_accel; + + run_kernel (hashcat_ctx, device_param, KERN_RUN_4, kernel_power_try, true, 0); + } } else { diff --git a/src/combinator.c b/src/combinator.c index 1e0c1f98b..5ca42fb87 100644 --- a/src/combinator.c +++ b/src/combinator.c @@ -33,7 +33,142 @@ int combinator_ctx_init (hashcat_ctx_t *hashcat_ctx) combinator_ctx->scratch_buf = (char *) hcmalloc (HCBUFSIZ_LARGE); - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) + { + if (user_options->attack_mode == ATTACK_MODE_COMBI) + { + // display + + char *dictfile1 = user_options_extra->hc_workv[0]; + char *dictfile2 = user_options_extra->hc_workv[1]; + + // at this point we know the file actually exist + // find the bigger dictionary and use as base + + if (hc_path_is_file (dictfile1) == false) + { + event_log_error (hashcat_ctx, "%s: Not a regular file.", dictfile1); + + return -1; + } + + if (hc_path_is_file (dictfile2) == false) + { + event_log_error (hashcat_ctx, "%s: Not a regular file.", dictfile2); + + return -1; + } + + FILE *fp1 = NULL; + FILE *fp2 = NULL; + + if ((fp1 = fopen (dictfile1, "rb")) == NULL) + { + event_log_error (hashcat_ctx, "%s: %s", dictfile1, strerror (errno)); + + return -1; + } + + if ((fp2 = fopen (dictfile2, "rb")) == NULL) + { + event_log_error (hashcat_ctx, "%s: %s", dictfile2, strerror (errno)); + + fclose (fp1); + + return -1; + } + + combinator_ctx->combs_cnt = 1; + + u64 words1_cnt = 0; + + const int rc1 = count_words (hashcat_ctx, fp1, dictfile1, &words1_cnt); + + if (rc1 == -1) + { + event_log_error (hashcat_ctx, "Integer overflow detected in keyspace of wordlist: %s", dictfile1); + + fclose (fp1); + fclose (fp2); + + return -1; + } + + if (words1_cnt == 0) + { + event_log_error (hashcat_ctx, "%s: empty file.", dictfile1); + + fclose (fp1); + fclose (fp2); + + return -1; + } + + combinator_ctx->combs_cnt = 1; + + u64 words2_cnt = 0; + + const int rc2 = count_words (hashcat_ctx, fp2, dictfile2, &words2_cnt); + + if (rc2 == -1) + { + event_log_error (hashcat_ctx, "Integer overflow detected in keyspace of wordlist: %s", dictfile2); + + fclose (fp1); + fclose (fp2); + + return -1; + } + + if (words2_cnt == 0) + { + event_log_error (hashcat_ctx, "%s: empty file.", dictfile2); + + fclose (fp1); + fclose (fp2); + + return -1; + } + + fclose (fp1); + fclose (fp2); + + combinator_ctx->dict1 = dictfile1; + combinator_ctx->dict2 = dictfile2; + + if (words1_cnt >= words2_cnt) + { + combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_LEFT; + combinator_ctx->combs_cnt = words2_cnt; + } + else + { + combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_RIGHT; + combinator_ctx->combs_cnt = words1_cnt; + + // we also have to switch wordlist related rules! + + char *tmpc = user_options->rule_buf_l; + + user_options->rule_buf_l = user_options->rule_buf_r; + user_options->rule_buf_r = tmpc; + + u32 tmpi = user_options_extra->rule_len_l; + + user_options_extra->rule_len_l = user_options_extra->rule_len_r; + user_options_extra->rule_len_r = tmpi; + } + } + else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) + { + combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_LEFT; + } + else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) + { + combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_RIGHT; + } + } + else { // this is always need to be COMBINATOR_MODE_BASE_LEFT @@ -190,141 +325,6 @@ int combinator_ctx_init (hashcat_ctx_t *hashcat_ctx) combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_LEFT; } } - else - { - if (user_options->attack_mode == ATTACK_MODE_COMBI) - { - // display - - char *dictfile1 = user_options_extra->hc_workv[0]; - char *dictfile2 = user_options_extra->hc_workv[1]; - - // at this point we know the file actually exist - // find the bigger dictionary and use as base - - if (hc_path_is_file (dictfile1) == false) - { - event_log_error (hashcat_ctx, "%s: Not a regular file.", dictfile1); - - return -1; - } - - if (hc_path_is_file (dictfile2) == false) - { - event_log_error (hashcat_ctx, "%s: Not a regular file.", dictfile2); - - return -1; - } - - FILE *fp1 = NULL; - FILE *fp2 = NULL; - - if ((fp1 = fopen (dictfile1, "rb")) == NULL) - { - event_log_error (hashcat_ctx, "%s: %s", dictfile1, strerror (errno)); - - return -1; - } - - if ((fp2 = fopen (dictfile2, "rb")) == NULL) - { - event_log_error (hashcat_ctx, "%s: %s", dictfile2, strerror (errno)); - - fclose (fp1); - - return -1; - } - - combinator_ctx->combs_cnt = 1; - - u64 words1_cnt = 0; - - const int rc1 = count_words (hashcat_ctx, fp1, dictfile1, &words1_cnt); - - if (rc1 == -1) - { - event_log_error (hashcat_ctx, "Integer overflow detected in keyspace of wordlist: %s", dictfile1); - - fclose (fp1); - fclose (fp2); - - return -1; - } - - if (words1_cnt == 0) - { - event_log_error (hashcat_ctx, "%s: empty file.", dictfile1); - - fclose (fp1); - fclose (fp2); - - return -1; - } - - combinator_ctx->combs_cnt = 1; - - u64 words2_cnt = 0; - - const int rc2 = count_words (hashcat_ctx, fp2, dictfile2, &words2_cnt); - - if (rc2 == -1) - { - event_log_error (hashcat_ctx, "Integer overflow detected in keyspace of wordlist: %s", dictfile2); - - fclose (fp1); - fclose (fp2); - - return -1; - } - - if (words2_cnt == 0) - { - event_log_error (hashcat_ctx, "%s: empty file.", dictfile2); - - fclose (fp1); - fclose (fp2); - - return -1; - } - - fclose (fp1); - fclose (fp2); - - combinator_ctx->dict1 = dictfile1; - combinator_ctx->dict2 = dictfile2; - - if (words1_cnt >= words2_cnt) - { - combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_LEFT; - combinator_ctx->combs_cnt = words2_cnt; - } - else - { - combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_RIGHT; - combinator_ctx->combs_cnt = words1_cnt; - - // we also have to switch wordlist related rules! - - char *tmpc = user_options->rule_buf_l; - - user_options->rule_buf_l = user_options->rule_buf_r; - user_options->rule_buf_r = tmpc; - - u32 tmpi = user_options_extra->rule_len_l; - - user_options_extra->rule_len_l = user_options_extra->rule_len_r; - user_options_extra->rule_len_r = tmpi; - } - } - else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) - { - combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_LEFT; - } - else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) - { - combinator_ctx->combs_mode = COMBINATOR_MODE_BASE_RIGHT; - } - } return 0; } diff --git a/src/dispatch.c b/src/dispatch.c index 8c9d03e79..f2ff388b3 100644 --- a/src/dispatch.c +++ b/src/dispatch.c @@ -313,9 +313,9 @@ static int calc (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param) const u32 attack_mode = user_options->attack_mode; const u32 attack_kern = user_options_extra->attack_kern; - if ((attack_mode == ATTACK_MODE_BF) || ((user_options->length_limit_disable == true) && (attack_mode == ATTACK_MODE_HYBRID2))) + if ((attack_mode == ATTACK_MODE_BF) || ((user_options->optimized_kernel_enable == false) && (attack_mode == ATTACK_MODE_HYBRID2))) { - if ((user_options->length_limit_disable == true) && (attack_mode == ATTACK_MODE_HYBRID2)) + if ((user_options->optimized_kernel_enable == false) && (attack_mode == ATTACK_MODE_HYBRID2)) { char *dictfile = straight_ctx->dict; diff --git a/src/interface.c b/src/interface.c index 29f762afc..d1ffc474a 100644 --- a/src/interface.c +++ b/src/interface.c @@ -24293,7 +24293,7 @@ int hashconfig_init (hashcat_ctx_t *hashcat_ctx) hashconfig->opts_type |= OPTS_TYPE_PT_NEVERCRACK; } - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == false) { hashconfig->opts_type &= ~OPTS_TYPE_PT_UTF16LE; hashconfig->opts_type &= ~OPTS_TYPE_PT_UTF16BE; @@ -24556,15 +24556,7 @@ int hashconfig_init (hashcat_ctx_t *hashcat_ctx) // pw_max : some algo suffer from support for long passwords, // the user need to add -L to enable support for them - if (user_options->length_limit_disable == true) - { - switch (hashconfig->hash_mode) - { - case 10700: hashconfig->pw_max = 127; // https://www.pdflib.com/knowledge-base/pdf-password-security/encryption/ - break; - } - } - else + if (user_options->optimized_kernel_enable == true) { hashconfig->pw_max = PW_MAX_OLD; @@ -24629,6 +24621,14 @@ int hashconfig_init (hashcat_ctx_t *hashcat_ctx) break; } } + else + { + switch (hashconfig->hash_mode) + { + case 10700: hashconfig->pw_max = 127; // https://www.pdflib.com/knowledge-base/pdf-password-security/encryption/ + break; + } + } // pw_max : all modes listed in the following switch cases are // the maximum possible password length by the related system @@ -24755,11 +24755,7 @@ int hashconfig_init (hashcat_ctx_t *hashcat_ctx) hashconfig->salt_min = SALT_MIN; hashconfig->salt_max = SALT_MAX; - if (user_options->length_limit_disable == true) - { - // nothing to do - } - else + if (user_options->optimized_kernel_enable == true) { hashconfig->salt_max = SALT_MAX_OLD; diff --git a/src/mpsp.c b/src/mpsp.c index 6ec2f127b..83f4ab1af 100644 --- a/src/mpsp.c +++ b/src/mpsp.c @@ -1174,7 +1174,7 @@ int mask_ctx_update_loop (hashcat_ctx_t *hashcat_ctx) } else if ((user_options->attack_mode == ATTACK_MODE_HYBRID1) || (user_options->attack_mode == ATTACK_MODE_HYBRID2)) { - if ((user_options->length_limit_disable == true) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) + if ((user_options->optimized_kernel_enable == false) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) { mask_ctx->mask = mask_ctx->masks[mask_ctx->masks_pos]; diff --git a/src/opencl.c b/src/opencl.c index cfe58710b..ac6c66c47 100644 --- a/src/opencl.c +++ b/src/opencl.c @@ -108,22 +108,22 @@ static int ocl_check_dri (MAYBE_UNUSED hashcat_ctx_t *hashcat_ctx) return 0; } -static void generate_source_kernel_filename (const u32 attack_exec, const u32 attack_kern, const u32 kern_type, const bool length_limit_disable, char *shared_dir, char *source_file) +static void generate_source_kernel_filename (const u32 attack_exec, const u32 attack_kern, const u32 kern_type, const bool optimized_kernel_enable, char *shared_dir, char *source_file) { - if (length_limit_disable == true) + if (optimized_kernel_enable == true) { if (attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { if (attack_kern == ATTACK_KERN_STRAIGHT) - snprintf (source_file, 255, "%s/OpenCL/m%05d_a0-pure.cl", shared_dir, (int) kern_type); + snprintf (source_file, 255, "%s/OpenCL/m%05d_a0-optimized.cl", shared_dir, (int) kern_type); else if (attack_kern == ATTACK_KERN_COMBI) - snprintf (source_file, 255, "%s/OpenCL/m%05d_a1-pure.cl", shared_dir, (int) kern_type); + snprintf (source_file, 255, "%s/OpenCL/m%05d_a1-optimized.cl", shared_dir, (int) kern_type); else if (attack_kern == ATTACK_KERN_BF) - snprintf (source_file, 255, "%s/OpenCL/m%05d_a3-pure.cl", shared_dir, (int) kern_type); + snprintf (source_file, 255, "%s/OpenCL/m%05d_a3-optimized.cl", shared_dir, (int) kern_type); } else { - snprintf (source_file, 255, "%s/OpenCL/m%05d-pure.cl", shared_dir, (int) kern_type); + snprintf (source_file, 255, "%s/OpenCL/m%05d-optimized.cl", shared_dir, (int) kern_type); } } else @@ -144,22 +144,22 @@ static void generate_source_kernel_filename (const u32 attack_exec, const u32 at } } -static void generate_cached_kernel_filename (const u32 attack_exec, const u32 attack_kern, const u32 kern_type, const bool length_limit_disable, char *profile_dir, const char *device_name_chksum, char *cached_file) +static void generate_cached_kernel_filename (const u32 attack_exec, const u32 attack_kern, const u32 kern_type, const bool optimized_kernel_enable, char *profile_dir, const char *device_name_chksum, char *cached_file) { - if (length_limit_disable == true) + if (optimized_kernel_enable == true) { if (attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { if (attack_kern == ATTACK_KERN_STRAIGHT) - snprintf (cached_file, 255, "%s/kernels/m%05d_a0-pure.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); + snprintf (cached_file, 255, "%s/kernels/m%05d_a0-optimized.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); else if (attack_kern == ATTACK_KERN_COMBI) - snprintf (cached_file, 255, "%s/kernels/m%05d_a1-pure.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); + snprintf (cached_file, 255, "%s/kernels/m%05d_a1-optimized.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); else if (attack_kern == ATTACK_KERN_BF) - snprintf (cached_file, 255, "%s/kernels/m%05d_a3-pure.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); + snprintf (cached_file, 255, "%s/kernels/m%05d_a3-optimized.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); } else { - snprintf (cached_file, 255, "%s/kernels/m%05d-pure.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); + snprintf (cached_file, 255, "%s/kernels/m%05d-optimized.%s.kernel", profile_dir, (int) kern_type, device_name_chksum); } } else @@ -1165,13 +1165,7 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, } } - if (user_options->length_limit_disable == true) - { - CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, pws_cnt, true, fast_iteration); - - if (CL_rc == -1) return -1; - } - else + if (user_options->optimized_kernel_enable == true) { if (highest_pw_len < 16) { @@ -1192,6 +1186,12 @@ int choose_kernel (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, if (CL_rc == -1) return -1; } } + else + { + CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, pws_cnt, true, fast_iteration); + + if (CL_rc == -1) return -1; + } } else { @@ -1792,32 +1792,7 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { - if (user_options->length_limit_disable == true) - { - if (user_options->attack_mode == ATTACK_MODE_COMBI) - { - const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_pws_buf, CL_TRUE, 0, pws_cnt * sizeof (pw_t), device_param->pws_buf, 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } - else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) - { - const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_pws_buf, CL_TRUE, 0, pws_cnt * sizeof (pw_t), device_param->pws_buf, 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } - else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) - { - const u64 off = device_param->words_off; - - device_param->kernel_params_mp_buf64[3] = off; - - const int CL_rc = run_kernel_mp (hashcat_ctx, device_param, KERN_RUN_MP, pws_cnt); - - if (CL_rc == -1) return -1; - } - } - else + if (user_options->optimized_kernel_enable == true) { if (user_options->attack_mode == ATTACK_MODE_COMBI) { @@ -1877,6 +1852,31 @@ int run_copy (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, const if (CL_rc == -1) return -1; } + else + { + if (user_options->attack_mode == ATTACK_MODE_COMBI) + { + const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_pws_buf, CL_TRUE, 0, pws_cnt * sizeof (pw_t), device_param->pws_buf, 0, NULL, NULL); + + if (CL_rc == -1) return -1; + } + else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) + { + const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_pws_buf, CL_TRUE, 0, pws_cnt * sizeof (pw_t), device_param->pws_buf, 0, NULL, NULL); + + if (CL_rc == -1) return -1; + } + else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) + { + const u64 off = device_param->words_off; + + device_param->kernel_params_mp_buf64[3] = off; + + const int CL_rc = run_kernel_mp (hashcat_ctx, device_param, KERN_RUN_MP, pws_cnt); + + if (CL_rc == -1) return -1; + } + } } else if (user_options_extra->attack_kern == ATTACK_KERN_BF) { @@ -1954,7 +1954,7 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co FILE *combs_fp = device_param->combs_fp; - if ((user_options->attack_mode == ATTACK_MODE_COMBI) || ((user_options->length_limit_disable == true) && (user_options->attack_mode == ATTACK_MODE_HYBRID2))) + if ((user_options->attack_mode == ATTACK_MODE_COMBI) || ((user_options->optimized_kernel_enable == false) && (user_options->attack_mode == ATTACK_MODE_HYBRID2))) { rewind (combs_fp); } @@ -2018,108 +2018,7 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { - if (user_options->length_limit_disable == true) - { - if ((user_options->attack_mode == ATTACK_MODE_COMBI) || (user_options->attack_mode == ATTACK_MODE_HYBRID2)) - { - char *line_buf = combinator_ctx->scratch_buf; - - u32 i = 0; - - while (i < innerloop_left) - { - if (feof (combs_fp)) break; - - int line_len = fgetl (combs_fp, line_buf); - - line_len = convert_from_hex (hashcat_ctx, line_buf, line_len); - - if (line_len >= PW_MAX) continue; - - char *line_buf_new = line_buf; - - char rule_buf_out[BLOCK_SIZE]; - - if (run_rule_engine (user_options_extra->rule_len_r, user_options->rule_buf_r)) - { - if (line_len >= BLOCK_SIZE) continue; - - memset (rule_buf_out, 0, sizeof (rule_buf_out)); - - const int rule_len_out = _old_apply_rule (user_options->rule_buf_r, user_options_extra->rule_len_r, line_buf, line_len, rule_buf_out); - - if (rule_len_out < 0) - { - status_ctx->words_progress_rejected[salt_pos] += pws_cnt; - - continue; - } - - line_len = rule_len_out; - - line_buf_new = rule_buf_out; - } - - line_len = MIN (line_len, PW_MAX - 1); - - u8 *ptr = (u8 *) device_param->combs_buf[i].i; - - memcpy (ptr, line_buf_new, line_len); - - memset (ptr + line_len, 0, PW_MAX - line_len); - - if (hashconfig->opts_type & OPTS_TYPE_PT_UPPER) - { - uppercase (ptr, line_len); - } - - if (combinator_ctx->combs_mode == COMBINATOR_MODE_BASE_LEFT) - { - if (hashconfig->opts_type & OPTS_TYPE_PT_ADD80) - { - ptr[line_len] = 0x80; - } - - if (hashconfig->opts_type & OPTS_TYPE_PT_ADD01) - { - ptr[line_len] = 0x01; - } - } - - device_param->combs_buf[i].pw_len = line_len; - - i++; - } - - for (u32 j = i; j < innerloop_left; j++) - { - memset (&device_param->combs_buf[j], 0, sizeof (pw_t)); - } - - innerloop_left = i; - - const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_combs_c, CL_TRUE, 0, innerloop_left * sizeof (pw_t), device_param->combs_buf, 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } - else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) - { - u64 off = innerloop_pos; - - device_param->kernel_params_mp_buf64[3] = off; - - int CL_rc; - - CL_rc = run_kernel_mp (hashcat_ctx, device_param, KERN_RUN_MP, innerloop_left); - - if (CL_rc == -1) return -1; - - CL_rc = hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->command_queue, device_param->d_combs, device_param->d_combs_c, 0, 0, innerloop_left * sizeof (pw_t), 0, NULL, NULL); - - if (CL_rc == -1) return -1; - } - } - else + if (user_options->optimized_kernel_enable == true) { if (user_options->attack_mode == ATTACK_MODE_COMBI) { @@ -2233,6 +2132,107 @@ int run_cracker (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, co CL_rc = hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->command_queue, device_param->d_combs, device_param->d_combs_c, 0, 0, innerloop_left * sizeof (pw_t), 0, NULL, NULL); + if (CL_rc == -1) return -1; + } + } + else + { + if ((user_options->attack_mode == ATTACK_MODE_COMBI) || (user_options->attack_mode == ATTACK_MODE_HYBRID2)) + { + char *line_buf = combinator_ctx->scratch_buf; + + u32 i = 0; + + while (i < innerloop_left) + { + if (feof (combs_fp)) break; + + int line_len = fgetl (combs_fp, line_buf); + + line_len = convert_from_hex (hashcat_ctx, line_buf, line_len); + + if (line_len >= PW_MAX) continue; + + char *line_buf_new = line_buf; + + char rule_buf_out[BLOCK_SIZE]; + + if (run_rule_engine (user_options_extra->rule_len_r, user_options->rule_buf_r)) + { + if (line_len >= BLOCK_SIZE) continue; + + memset (rule_buf_out, 0, sizeof (rule_buf_out)); + + const int rule_len_out = _old_apply_rule (user_options->rule_buf_r, user_options_extra->rule_len_r, line_buf, line_len, rule_buf_out); + + if (rule_len_out < 0) + { + status_ctx->words_progress_rejected[salt_pos] += pws_cnt; + + continue; + } + + line_len = rule_len_out; + + line_buf_new = rule_buf_out; + } + + line_len = MIN (line_len, PW_MAX - 1); + + u8 *ptr = (u8 *) device_param->combs_buf[i].i; + + memcpy (ptr, line_buf_new, line_len); + + memset (ptr + line_len, 0, PW_MAX - line_len); + + if (hashconfig->opts_type & OPTS_TYPE_PT_UPPER) + { + uppercase (ptr, line_len); + } + + if (combinator_ctx->combs_mode == COMBINATOR_MODE_BASE_LEFT) + { + if (hashconfig->opts_type & OPTS_TYPE_PT_ADD80) + { + ptr[line_len] = 0x80; + } + + if (hashconfig->opts_type & OPTS_TYPE_PT_ADD01) + { + ptr[line_len] = 0x01; + } + } + + device_param->combs_buf[i].pw_len = line_len; + + i++; + } + + for (u32 j = i; j < innerloop_left; j++) + { + memset (&device_param->combs_buf[j], 0, sizeof (pw_t)); + } + + innerloop_left = i; + + const int CL_rc = hc_clEnqueueWriteBuffer (hashcat_ctx, device_param->command_queue, device_param->d_combs_c, CL_TRUE, 0, innerloop_left * sizeof (pw_t), device_param->combs_buf, 0, NULL, NULL); + + if (CL_rc == -1) return -1; + } + else if (user_options->attack_mode == ATTACK_MODE_HYBRID1) + { + u64 off = innerloop_pos; + + device_param->kernel_params_mp_buf64[3] = off; + + int CL_rc; + + CL_rc = run_kernel_mp (hashcat_ctx, device_param, KERN_RUN_MP, innerloop_left); + + if (CL_rc == -1) return -1; + + CL_rc = hc_clEnqueueCopyBuffer (hashcat_ctx, device_param->command_queue, device_param->d_combs, device_param->d_combs_c, 0, 0, innerloop_left * sizeof (pw_t), 0, NULL, NULL); + if (CL_rc == -1) return -1; } } @@ -3728,13 +3728,17 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) vector_width = user_options->opencl_vector_width; } - // We can't have SIMD in kernels where final password length depends on user input we can't precompute + // We can't have SIMD in kernels where we have an unknown final password length - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == false) { if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { - if ((user_options_extra->attack_kern == ATTACK_KERN_STRAIGHT) || (user_options_extra->attack_kern == ATTACK_KERN_COMBI)) + if (user_options_extra->attack_kern == ATTACK_KERN_STRAIGHT) + { + vector_width = 1; + } + else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { vector_width = 1; } @@ -4260,13 +4264,32 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) char source_file[256] = { 0 }; - generate_source_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, user_options->length_limit_disable, folder_config->shared_dir, source_file); + generate_source_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, user_options->optimized_kernel_enable, folder_config->shared_dir, source_file); - if (hc_path_read (source_file) == false) + if (user_options->optimized_kernel_enable == true) { - event_log_error (hashcat_ctx, "%s: %s", source_file, strerror (errno)); + if (hc_path_read (source_file) == false) + { + if (user_options->quiet == false) event_log_warning (hashcat_ctx, "%s: Optimized kernel not found, falling back to pure kernel", source_file); - return -1; + generate_source_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, false, folder_config->shared_dir, source_file); + } + + if (hc_path_read (source_file) == false) + { + event_log_error (hashcat_ctx, "%s: %s", source_file, strerror (errno)); + + return -1; + } + } + else + { + if (hc_path_read (source_file) == false) + { + event_log_error (hashcat_ctx, "%s: %s", source_file, strerror (errno)); + + return -1; + } } /** @@ -4275,18 +4298,43 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) char cached_file[256] = { 0 }; - generate_cached_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, user_options->length_limit_disable, folder_config->profile_dir, device_name_chksum, cached_file); + generate_cached_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, user_options->optimized_kernel_enable, folder_config->profile_dir, device_name_chksum, cached_file); bool cached = true; - if (hc_path_read (cached_file) == false) + if (user_options->optimized_kernel_enable == true) { - cached = false; - } + if (hc_path_read (cached_file) == false) + { + generate_cached_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, false, folder_config->profile_dir, device_name_chksum, cached_file); + } - if (hc_path_is_empty (cached_file) == true) + if (hc_path_is_empty (cached_file) == true) + { + generate_cached_kernel_filename (hashconfig->attack_exec, user_options_extra->attack_kern, hashconfig->kern_type, false, folder_config->profile_dir, device_name_chksum, cached_file); + } + + if (hc_path_read (cached_file) == false) + { + cached = false; + } + + if (hc_path_is_empty (cached_file) == true) + { + cached = false; + } + } + else { - cached = false; + if (hc_path_read (cached_file) == false) + { + cached = false; + } + + if (hc_path_is_empty (cached_file) == true) + { + cached = false; + } } /** @@ -4919,7 +4967,11 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) device_param->kernel_params_mp_buf32[7] = 0; device_param->kernel_params_mp_buf32[8] = 0; - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) + { + device_param->kernel_params_mp[0] = &device_param->d_combs; + } + else { if (user_options->attack_mode == ATTACK_MODE_HYBRID1) { @@ -4930,10 +4982,6 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) device_param->kernel_params_mp[0] = &device_param->d_pws_buf; } } - else - { - device_param->kernel_params_mp[0] = &device_param->d_combs; - } device_param->kernel_params_mp[1] = &device_param->d_root_css_buf; device_param->kernel_params_mp[2] = &device_param->d_markov_css_buf; @@ -5011,19 +5059,7 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) { if (hashconfig->opti_type & OPTI_TYPE_SINGLE_HASH) { - if (user_options->length_limit_disable == true) - { - snprintf (kernel_name, sizeof (kernel_name) - 1, "m%05u_sxx", hashconfig->kern_type); - - CL_rc = hc_clCreateKernel (hashcat_ctx, device_param->program, kernel_name, &device_param->kernel4); - - if (CL_rc == -1) return -1; - - CL_rc = get_kernel_threads (hashcat_ctx, device_param, device_param->kernel4, &device_param->kernel_threads_by_wgs_kernel4); - - if (CL_rc == -1) return -1; - } - else + if (user_options->optimized_kernel_enable == true) { // kernel1 @@ -5061,12 +5097,9 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) if (CL_rc == -1) return -1; } - } - else - { - if (user_options->length_limit_disable == true) + else { - snprintf (kernel_name, sizeof (kernel_name) - 1, "m%05u_mxx", hashconfig->kern_type); + snprintf (kernel_name, sizeof (kernel_name) - 1, "m%05u_sxx", hashconfig->kern_type); CL_rc = hc_clCreateKernel (hashcat_ctx, device_param->program, kernel_name, &device_param->kernel4); @@ -5076,7 +5109,10 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) if (CL_rc == -1) return -1; } - else + } + else + { + if (user_options->optimized_kernel_enable == true) { // kernel1 @@ -5112,6 +5148,18 @@ int opencl_session_begin (hashcat_ctx_t *hashcat_ctx) CL_rc = get_kernel_threads (hashcat_ctx, device_param, device_param->kernel3, &device_param->kernel_threads_by_wgs_kernel3); + if (CL_rc == -1) return -1; + } + else + { + snprintf (kernel_name, sizeof (kernel_name) - 1, "m%05u_mxx", hashconfig->kern_type); + + CL_rc = hc_clCreateKernel (hashcat_ctx, device_param->program, kernel_name, &device_param->kernel4); + + if (CL_rc == -1) return -1; + + CL_rc = get_kernel_threads (hashcat_ctx, device_param, device_param->kernel4, &device_param->kernel_threads_by_wgs_kernel4); + if (CL_rc == -1) return -1; } } diff --git a/src/outfile.c b/src/outfile.c index da20ac9cc..a03051fc4 100644 --- a/src/outfile.c +++ b/src/outfile.c @@ -86,7 +86,7 @@ int build_plain (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl plain_len += comb_len; - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { int pw_max_combi; @@ -152,33 +152,7 @@ int build_plain (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) - { - pw_t pw; - - const int rc = gidd_to_pw_t (hashcat_ctx, device_param, gidvid, &pw); - - if (rc == -1) return -1; - - u64 off = device_param->kernel_params_mp_buf64[3] + gidvid; - - u32 start = 0; - u32 stop = device_param->kernel_params_mp_buf32[4]; - - sp_exec (off, (char *) plain_ptr, mask_ctx->root_css_buf, mask_ctx->markov_css_buf, start, start + stop); - - plain_len = stop; - - char *comb_buf = (char *) device_param->combs_buf[il_pos].i; - u32 comb_len = device_param->combs_buf[il_pos].pw_len; - - memcpy (plain_ptr + plain_len, comb_buf, comb_len); - - plain_len += comb_len; - - if (plain_len > (int) hashconfig->pw_max) plain_len = (int) hashconfig->pw_max; - } - else + if (user_options->optimized_kernel_enable == true) { pw_t pw; @@ -204,6 +178,32 @@ int build_plain (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param, pl plain_len += start + stop; + if (plain_len > (int) hashconfig->pw_max) plain_len = (int) hashconfig->pw_max; + } + else + { + pw_t pw; + + const int rc = gidd_to_pw_t (hashcat_ctx, device_param, gidvid, &pw); + + if (rc == -1) return -1; + + u64 off = device_param->kernel_params_mp_buf64[3] + gidvid; + + u32 start = 0; + u32 stop = device_param->kernel_params_mp_buf32[4]; + + sp_exec (off, (char *) plain_ptr, mask_ctx->root_css_buf, mask_ctx->markov_css_buf, start, start + stop); + + plain_len = stop; + + char *comb_buf = (char *) device_param->combs_buf[il_pos].i; + u32 comb_len = device_param->combs_buf[il_pos].pw_len; + + memcpy (plain_ptr + plain_len, comb_buf, comb_len); + + plain_len += comb_len; + if (plain_len > (int) hashconfig->pw_max) plain_len = (int) hashconfig->pw_max; } } diff --git a/src/selftest.c b/src/selftest.c index 34b9bbce5..65315b88c 100644 --- a/src/selftest.c +++ b/src/selftest.c @@ -306,13 +306,7 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { - if (user_options->length_limit_disable == true) - { - CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, 1, false, 0); - - if (CL_rc == -1) return -1; - } - else + if (user_options->optimized_kernel_enable == true) { if (highest_pw_len < 16) { @@ -333,6 +327,12 @@ static int selftest (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (CL_rc == -1) return -1; } } + else + { + CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, 1, false, 0); + + if (CL_rc == -1) return -1; + } } else { diff --git a/src/status.c b/src/status.c index 548b47fe8..3e664c1f3 100644 --- a/src/status.c +++ b/src/status.c @@ -474,7 +474,7 @@ char *status_get_guess_base (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const mask_ctx_t *mask_ctx = hashcat_ctx->mask_ctx; @@ -519,7 +519,7 @@ int status_get_guess_base_offset (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const mask_ctx_t *mask_ctx = hashcat_ctx->mask_ctx; @@ -564,7 +564,7 @@ int status_get_guess_base_count (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const mask_ctx_t *mask_ctx = hashcat_ctx->mask_ctx; @@ -624,7 +624,7 @@ char *status_get_guess_mod (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; @@ -665,7 +665,7 @@ int status_get_guess_mod_offset (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; @@ -706,7 +706,7 @@ int status_get_guess_mod_count (const hashcat_ctx_t *hashcat_ctx) } else if (user_options->attack_mode == ATTACK_MODE_HYBRID2) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { const straight_ctx_t *straight_ctx = hashcat_ctx->straight_ctx; diff --git a/src/terminal.c b/src/terminal.c index 0805422f6..4264cf7f9 100644 --- a/src/terminal.c +++ b/src/terminal.c @@ -888,7 +888,7 @@ void status_display (hashcat_ctx_t *hashcat_ctx) case GUESS_MODE_HYBRID2: - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { event_log_info (hashcat_ctx, "Guess.Base.......: Mask (%s) [%d], Left Side", @@ -915,7 +915,7 @@ void status_display (hashcat_ctx_t *hashcat_ctx) case GUESS_MODE_HYBRID2_CS: - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { event_log_info (hashcat_ctx, "Guess.Base.......: Mask (%s) [%d], Left Side", diff --git a/src/user_options.c b/src/user_options.c index 90c375c37..93e7c020a 100644 --- a/src/user_options.c +++ b/src/user_options.c @@ -14,7 +14,7 @@ #include "outfile.h" #include "user_options.h" -static const char short_options[] = "hVvm:a:r:j:k:g:o:t:d:D:n:u:c:p:s:l:1:2:3:4:iIbw:L"; +static const char short_options[] = "hVvm:a:r:j:k:g:o:t:d:D:n:u:c:p:s:l:1:2:3:4:iIbw:O"; static const struct option long_options[] = { @@ -55,7 +55,6 @@ static const struct option long_options[] = {"kernel-loops", required_argument, 0, IDX_KERNEL_LOOPS}, {"keyspace", no_argument, 0, IDX_KEYSPACE}, {"left", no_argument, 0, IDX_LEFT}, - {"length-limit-disable", no_argument, 0, IDX_LENGTH_LIMIT_DISABLE}, {"limit", required_argument, 0, IDX_LIMIT}, {"logfile-disable", no_argument, 0, IDX_LOGFILE_DISABLE}, {"loopback", no_argument, 0, IDX_LOOPBACK}, @@ -71,6 +70,7 @@ static const struct option long_options[] = {"opencl-info", no_argument, 0, IDX_OPENCL_INFO}, {"opencl-platforms", required_argument, 0, IDX_OPENCL_PLATFORMS}, {"opencl-vector-width", required_argument, 0, IDX_OPENCL_VECTOR_WIDTH}, + {"optimized-kernel-enable", no_argument, 0, IDX_OPTIMIZED_KERNEL_ENABLE}, {"outfile-autohex-disable", no_argument, 0, IDX_OUTFILE_AUTOHEX_DISABLE}, {"outfile-check-dir", required_argument, 0, IDX_OUTFILE_CHECK_DIR}, {"outfile-check-timer", required_argument, 0, IDX_OUTFILE_CHECK_TIMER}, @@ -158,7 +158,6 @@ int user_options_init (hashcat_ctx_t *hashcat_ctx) user_options->keep_guessing = KEEP_GUESSING; user_options->keyspace = KEYSPACE; user_options->left = LEFT; - user_options->length_limit_disable = LENGTH_LIMIT_DISABLE; user_options->limit = LIMIT; user_options->logfile_disable = LOGFILE_DISABLE; user_options->loopback = LOOPBACK; @@ -174,6 +173,7 @@ int user_options_init (hashcat_ctx_t *hashcat_ctx) user_options->opencl_info = 0; user_options->opencl_platforms = NULL; user_options->opencl_vector_width = OPENCL_VECTOR_WIDTH; + user_options->optimized_kernel_enable = OPTIMIZED_KERNEL_ENABLE; user_options->outfile_autohex = OUTFILE_AUTOHEX; user_options->outfile_check_dir = NULL; user_options->outfile_check_timer = OUTFILE_CHECK_TIMER; @@ -316,7 +316,6 @@ int user_options_getopt (hashcat_ctx_t *hashcat_ctx, int argc, char **argv) case IDX_QUIET: user_options->quiet = true; break; case IDX_SHOW: user_options->show = true; break; case IDX_LEFT: user_options->left = true; break; - case IDX_LENGTH_LIMIT_DISABLE: user_options->length_limit_disable = true; break; case IDX_ADVICE_DISABLE: user_options->advice_disable = true; break; case IDX_USERNAME: user_options->username = true; break; case IDX_REMOVE: user_options->remove = true; break; @@ -382,6 +381,7 @@ int user_options_getopt (hashcat_ctx_t *hashcat_ctx, int argc, char **argv) case IDX_OPENCL_DEVICE_TYPES: user_options->opencl_device_types = optarg; break; case IDX_OPENCL_VECTOR_WIDTH: user_options->opencl_vector_width = atoi (optarg); user_options->opencl_vector_width_chgd = true; break; + case IDX_OPTIMIZED_KERNEL_ENABLE: user_options->optimized_kernel_enable = true; break; case IDX_WORKLOAD_PROFILE: user_options->workload_profile = atoi (optarg); user_options->workload_profile_chgd = true; break; case IDX_KERNEL_ACCEL: user_options->kernel_accel = atoi (optarg); @@ -613,14 +613,14 @@ int user_options_sanity (hashcat_ctx_t *hashcat_ctx) return -1; } - if (user_options->rp_files_cnt > 0 && user_options->rp_gen == true) + if ((user_options->rp_files_cnt > 0) && (user_options->rp_gen > 0)) { event_log_error (hashcat_ctx, "Combining -r/--rules-file and -g/--rules-generate is not supported."); return -1; } - if (user_options->rp_files_cnt > 0 || user_options->rp_gen == true) + if ((user_options->rp_files_cnt > 0) || (user_options->rp_gen > 0)) { if (user_options->attack_mode != ATTACK_MODE_STRAIGHT) { @@ -1266,49 +1266,50 @@ void user_options_preprocess (hashcat_ctx_t *hashcat_ctx) user_options->nvidia_spin_damp = 0; user_options->potfile_disable = true; user_options->powertune_enable = true; + user_options->progress_only = false; user_options->restore_disable = true; user_options->restore = false; user_options->restore_timer = 0; user_options->show = false; + user_options->speed_only = true; user_options->status = false; user_options->status_timer = 0; - user_options->speed_only = true; - user_options->progress_only = false; user_options->weak_hash_threshold = 0; if (user_options->workload_profile_chgd == false) { - user_options->workload_profile = 3; + user_options->optimized_kernel_enable = true; + user_options->workload_profile = 3; } } if (user_options->progress_only == true) { - user_options->speed_only = true; + user_options->speed_only = true; } if (user_options->keyspace == true) { - user_options->quiet = true; + user_options->quiet = true; } if (user_options->stdout_flag == true) { - user_options->quiet = true; - user_options->hash_mode = 2000; - user_options->outfile_format = OUTFILE_FMT_PLAIN; user_options->force = true; + user_options->hash_mode = 2000; user_options->kernel_accel = 1024; user_options->kernel_loops = 1024; user_options->opencl_vector_width = 1; + user_options->outfile_format = OUTFILE_FMT_PLAIN; + user_options->quiet = true; } if (user_options->opencl_info == true) { - user_options->quiet = true; - user_options->opencl_platforms = NULL; user_options->opencl_devices = NULL; user_options->opencl_device_types = hcstrdup ("1,2,3"); + user_options->opencl_platforms = NULL; + user_options->quiet = true; } if (user_options->left == true) @@ -1420,6 +1421,16 @@ void user_options_preprocess (hashcat_ctx_t *hashcat_ctx) } } } + + // temp + + if ((user_options->rp_files_cnt > 0) || (user_options->rp_gen > 0)) + { + // pure (unoptimized) kernels do not yet have support for rules, switch to opimized mode + // we can remove this after rule engine has been converted + + user_options->optimized_kernel_enable = true; + } } void user_options_postprocess (hashcat_ctx_t *hashcat_ctx) @@ -2128,7 +2139,6 @@ void user_options_logger (hashcat_ctx_t *hashcat_ctx) logfile_top_uint (user_options->kernel_loops); logfile_top_uint (user_options->keyspace); logfile_top_uint (user_options->left); - logfile_top_uint (user_options->length_limit_disable); logfile_top_uint (user_options->logfile_disable); logfile_top_uint (user_options->loopback); logfile_top_uint (user_options->machine_readable); @@ -2138,6 +2148,7 @@ void user_options_logger (hashcat_ctx_t *hashcat_ctx) logfile_top_uint (user_options->nvidia_spin_damp); logfile_top_uint (user_options->opencl_info); logfile_top_uint (user_options->opencl_vector_width); + logfile_top_uint (user_options->optimized_kernel_enable); logfile_top_uint (user_options->outfile_autohex); logfile_top_uint (user_options->outfile_check_timer); logfile_top_uint (user_options->outfile_format); diff --git a/src/weak_hash.c b/src/weak_hash.c index 1de57815e..9396fbfe1 100644 --- a/src/weak_hash.c +++ b/src/weak_hash.c @@ -38,15 +38,15 @@ int weak_hash_check (hashcat_ctx_t *hashcat_ctx, hc_device_param_t *device_param if (hashconfig->attack_exec == ATTACK_EXEC_INSIDE_KERNEL) { - if (user_options->length_limit_disable == true) + if (user_options->optimized_kernel_enable == true) { - CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, 1, false, 0); + CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_1, 1, false, 0); if (CL_rc == -1) return -1; } else { - CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_1, 1, false, 0); + CL_rc = run_kernel (hashcat_ctx, device_param, KERN_RUN_4, 1, false, 0); if (CL_rc == -1) return -1; } diff --git a/src/wordlist.c b/src/wordlist.c index b09573d14..5a5ecfbac 100644 --- a/src/wordlist.c +++ b/src/wordlist.c @@ -321,7 +321,7 @@ int count_words (hashcat_ctx_t *hashcat_ctx, FILE *fd, const char *dictfile, u64 } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { - if ((user_options->length_limit_disable == true) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) + if ((user_options->optimized_kernel_enable == false) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) { if (overflow_check_u64_mul (keyspace, mask_ctx->bfs_cnt) == false) return -1; @@ -424,7 +424,7 @@ int count_words (hashcat_ctx_t *hashcat_ctx, FILE *fd, const char *dictfile, u64 } else if (user_options_extra->attack_kern == ATTACK_KERN_COMBI) { - if ((user_options->length_limit_disable == true) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) + if ((user_options->optimized_kernel_enable == false) && (user_options->attack_mode == ATTACK_MODE_HYBRID2)) { if (overflow_check_u64_add (cnt, mask_ctx->bfs_cnt) == false) return -1; diff --git a/tools/test.sh b/tools/test.sh index 5fb6f7a13..403633cec 100755 --- a/tools/test.sh +++ b/tools/test.sh @@ -24,7 +24,7 @@ NEVER_CRACK="11600 14900" SLOW_ALGOS="400 500 501 1600 1800 2100 2500 3200 5200 5800 6211 6212 6213 6221 6222 6223 6231 6232 6233 6241 6242 6243 6251 6261 6271 6281 6300 6400 6500 6600 6700 6800 7100 7200 7400 7900 8200 8800 8900 9000 9100 9200 9300 9400 9500 9600 10000 10300 10500 10700 10900 11300 11600 11900 12000 12001 12100 12200 12300 12400 12500 12700 12800 12900 13000 13200 13400 13600 14600 14700 14800 15100 15200 15300 15600 15700" -OPTS="--quiet --force --potfile-disable --runtime 400 --gpu-temp-disable --weak-hash-threshold=0 -L" +OPTS="--quiet --force --potfile-disable --runtime 400 --gpu-temp-disable --weak-hash-threshold=0" OUTD="test_$(date +%s)"